2024-11-24 20:45:39,674 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 20:45:39,684 main DEBUG Took 0.009063 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 20:45:39,685 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 20:45:39,685 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 20:45:39,690 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 20:45:39,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,697 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 20:45:39,709 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,711 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,712 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,713 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,714 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,714 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,715 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,715 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,716 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,716 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,717 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,717 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,718 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,718 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,719 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,719 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,720 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:45:39,720 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,721 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 20:45:39,722 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:45:39,723 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 20:45:39,725 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 20:45:39,725 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 20:45:39,727 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 20:45:39,727 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 20:45:39,734 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 20:45:39,736 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 20:45:39,738 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 20:45:39,738 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 20:45:39,739 main DEBUG createAppenders(={Console}) 2024-11-24 20:45:39,739 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-24 20:45:39,740 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 20:45:39,740 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-24 20:45:39,740 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 20:45:39,741 main DEBUG OutputStream closed 2024-11-24 20:45:39,741 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 20:45:39,741 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 20:45:39,741 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-24 20:45:39,809 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 20:45:39,811 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 20:45:39,812 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 20:45:39,813 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 20:45:39,813 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 20:45:39,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 20:45:39,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 20:45:39,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 20:45:39,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 20:45:39,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 20:45:39,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 20:45:39,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 20:45:39,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 20:45:39,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 20:45:39,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 20:45:39,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 20:45:39,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 20:45:39,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 20:45:39,820 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 20:45:39,820 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-24 20:45:39,821 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 20:45:39,821 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-24T20:45:39,833 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-24 20:45:39,836 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 20:45:39,836 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T20:45:40,058 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9 2024-11-24T20:45:40,087 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=552, ProcessCount=11, AvailableMemoryMB=11719 2024-11-24T20:45:40,201 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T20:45:40,299 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:40,318 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/cluster_72f23f30-d5c2-ff7d-e5a6-a827c5ce7452, deleteOnExit=true 2024-11-24T20:45:40,481 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/cluster_72f23f30-d5c2-ff7d-e5a6-a827c5ce7452/zookeeper_0, clientPort=60770, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/cluster_72f23f30-d5c2-ff7d-e5a6-a827c5ce7452/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/cluster_72f23f30-d5c2-ff7d-e5a6-a827c5ce7452/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T20:45:40,497 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60770 2024-11-24T20:45:40,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:40,602 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 with version=8 2024-11-24T20:45:40,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/hbase-staging 2024-11-24T20:45:40,705 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T20:45:41,001 INFO [Time-limited test {}] client.ConnectionUtils(128): master/20b981ef5c27:0 server-side Connection retries=45 2024-11-24T20:45:41,013 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:41,015 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:41,024 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T20:45:41,024 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:41,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T20:45:41,208 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T20:45:41,325 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T20:45:41,384 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T20:45:41,400 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T20:45:41,405 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T20:45:41,451 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 16481 (auto-detected) 2024-11-24T20:45:41,455 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T20:45:41,480 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36507 2024-11-24T20:45:41,507 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36507 connecting to ZooKeeper ensemble=127.0.0.1:60770 2024-11-24T20:45:41,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:365070x0, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T20:45:41,594 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36507-0x1016e9842920000 connected 2024-11-24T20:45:41,698 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:41,699 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:41,712 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:41,717 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99, hbase.cluster.distributed=false 2024-11-24T20:45:41,750 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T20:45:41,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36507 2024-11-24T20:45:41,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36507 2024-11-24T20:45:41,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36507 2024-11-24T20:45:41,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36507 2024-11-24T20:45:41,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36507 2024-11-24T20:45:41,877 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/20b981ef5c27:0 server-side Connection retries=45 2024-11-24T20:45:41,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:41,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:41,880 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T20:45:41,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:41,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T20:45:41,883 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T20:45:41,887 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T20:45:41,888 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40981 2024-11-24T20:45:41,890 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40981 connecting to ZooKeeper ensemble=127.0.0.1:60770 2024-11-24T20:45:41,891 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:41,892 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:41,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409810x0, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T20:45:41,904 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40981-0x1016e9842920001 connected 2024-11-24T20:45:41,904 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:409810x0, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:41,909 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T20:45:41,925 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/bucket.cache 2024-11-24T20:45:41,934 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T20:45:41,935 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T20:45:41,938 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T20:45:41,941 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T20:45:41,941 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T20:45:41,950 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T20:45:41,952 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T20:45:41,953 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/bucket.persistence is complete. 2024-11-24T20:45:41,957 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T20:45:41,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T20:45:41,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T20:45:41,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40981 2024-11-24T20:45:41,981 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40981 2024-11-24T20:45:41,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40981 2024-11-24T20:45:41,984 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40981 2024-11-24T20:45:41,984 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40981 2024-11-24T20:45:42,006 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/20b981ef5c27:0 server-side Connection retries=45 2024-11-24T20:45:42,006 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:42,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:42,007 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T20:45:42,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:42,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T20:45:42,007 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T20:45:42,008 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T20:45:42,013 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40419 2024-11-24T20:45:42,015 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40419 connecting to ZooKeeper ensemble=127.0.0.1:60770 2024-11-24T20:45:42,015 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:42,017 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:42,087 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404190x0, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T20:45:42,088 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40419-0x1016e9842920002 connected 2024-11-24T20:45:42,089 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:42,089 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T20:45:42,091 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/bucket.cache 2024-11-24T20:45:42,091 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T20:45:42,091 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T20:45:42,092 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T20:45:42,092 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T20:45:42,092 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T20:45:42,092 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T20:45:42,093 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T20:45:42,093 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/bucket.persistence is complete. 2024-11-24T20:45:42,094 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T20:45:42,096 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T20:45:42,099 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T20:45:42,104 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40419 2024-11-24T20:45:42,105 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40419 2024-11-24T20:45:42,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40419 2024-11-24T20:45:42,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40419 2024-11-24T20:45:42,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40419 2024-11-24T20:45:42,127 DEBUG [M:0;20b981ef5c27:36507 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;20b981ef5c27:36507 2024-11-24T20:45:42,128 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/20b981ef5c27,36507,1732481140790 2024-11-24T20:45:42,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:42,167 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:42,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:42,169 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/20b981ef5c27,36507,1732481140790 2024-11-24T20:45:42,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T20:45:42,208 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T20:45:42,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,208 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,209 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T20:45:42,210 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/20b981ef5c27,36507,1732481140790 from backup master directory 2024-11-24T20:45:42,225 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:42,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:42,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/20b981ef5c27,36507,1732481140790 2024-11-24T20:45:42,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:42,225 WARN [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T20:45:42,226 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=20b981ef5c27,36507,1732481140790 2024-11-24T20:45:42,228 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T20:45:42,229 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T20:45:42,237 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/hbase.id] with ID: 93158d97-66e5-41c8-88a4-7d9ba5e3dd52 2024-11-24T20:45:42,238 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/.tmp/hbase.id 2024-11-24T20:45:42,243 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/hbase.id] 2024-11-24T20:45:42,278 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:42,285 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T20:45:42,307 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-24T20:45:42,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,320 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,348 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:45:42,352 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T20:45:42,393 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:45:42,399 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:45:42,445 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store 2024-11-24T20:45:42,468 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T20:45:42,472 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:42,474 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T20:45:42,475 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:42,475 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:42,477 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T20:45:42,477 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:42,478 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:42,479 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732481142474Disabling compacts and flushes for region at 1732481142474Disabling writes for close at 1732481142477 (+3 ms)Writing region close event to WAL at 1732481142478 (+1 ms)Closed at 1732481142478 2024-11-24T20:45:42,479 WARN [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/.initializing 2024-11-24T20:45:42,480 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/WALs/20b981ef5c27,36507,1732481140790 2024-11-24T20:45:42,493 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T20:45:42,508 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=20b981ef5c27%2C36507%2C1732481140790, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/WALs/20b981ef5c27,36507,1732481140790, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/oldWALs, maxLogs=10 2024-11-24T20:45:42,545 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/WALs/20b981ef5c27,36507,1732481140790/20b981ef5c27%2C36507%2C1732481140790.1732481142513 2024-11-24T20:45:42,546 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T20:45:42,547 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:45:42,547 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:42,550 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,551 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,585 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T20:45:42,614 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:42,616 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:42,616 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T20:45:42,622 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:42,622 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:42,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T20:45:42,627 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:42,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:42,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T20:45:42,632 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:42,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:42,633 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,637 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,638 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,641 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,642 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,644 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T20:45:42,647 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:42,654 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:45:42,655 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59749824, jitterRate=-0.10965824127197266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T20:45:42,662 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732481142563Initializing all the Stores at 1732481142564 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481142564Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481142565 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481142565Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481142565Cleaning up temporary data from old regions at 1732481142642 (+77 ms)Region opened successfully at 1732481142662 (+20 ms) 2024-11-24T20:45:42,664 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T20:45:42,692 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604b4acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=20b981ef5c27/172.17.0.2:0 2024-11-24T20:45:42,721 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T20:45:42,733 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T20:45:42,733 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T20:45:42,736 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T20:45:42,736 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T20:45:42,742 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T20:45:42,742 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T20:45:42,766 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T20:45:42,775 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T20:45:42,829 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T20:45:42,832 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T20:45:42,834 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T20:45:42,845 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T20:45:42,848 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T20:45:42,853 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T20:45:42,866 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T20:45:42,869 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T20:45:42,881 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T20:45:42,903 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T20:45:42,913 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T20:45:42,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:42,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:42,934 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:42,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,935 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,940 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=20b981ef5c27,36507,1732481140790, sessionid=0x1016e9842920000, setting cluster-up flag (Was=false) 2024-11-24T20:45:42,976 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:42,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:43,008 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T20:45:43,013 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=20b981ef5c27,36507,1732481140790 2024-11-24T20:45:43,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:43,035 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:43,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:43,067 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T20:45:43,069 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=20b981ef5c27,36507,1732481140790 2024-11-24T20:45:43,074 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T20:45:43,115 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(746): ClusterId : 93158d97-66e5-41c8-88a4-7d9ba5e3dd52 2024-11-24T20:45:43,115 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(746): ClusterId : 93158d97-66e5-41c8-88a4-7d9ba5e3dd52 2024-11-24T20:45:43,117 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T20:45:43,117 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T20:45:43,143 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T20:45:43,153 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T20:45:43,159 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T20:45:43,164 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 20b981ef5c27,36507,1732481140790 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T20:45:43,172 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T20:45:43,172 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T20:45:43,172 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T20:45:43,172 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T20:45:43,172 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:45:43,173 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:45:43,173 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:45:43,173 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:45:43,173 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/20b981ef5c27:0, corePoolSize=10, maxPoolSize=10 2024-11-24T20:45:43,173 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,174 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/20b981ef5c27:0, corePoolSize=2, maxPoolSize=2 2024-11-24T20:45:43,174 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,175 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732481173175 2024-11-24T20:45:43,177 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T20:45:43,178 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T20:45:43,178 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T20:45:43,179 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T20:45:43,181 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T20:45:43,182 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T20:45:43,182 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T20:45:43,183 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T20:45:43,184 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:43,185 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T20:45:43,184 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,188 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T20:45:43,189 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T20:45:43,190 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T20:45:43,192 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T20:45:43,193 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T20:45:43,195 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.large.0-1732481143194,5,FailOnTimeoutGroup] 2024-11-24T20:45:43,195 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.small.0-1732481143195,5,FailOnTimeoutGroup] 2024-11-24T20:45:43,196 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,196 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T20:45:43,197 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,198 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,201 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T20:45:43,202 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 2024-11-24T20:45:43,210 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:43,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T20:45:43,213 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T20:45:43,213 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T20:45:43,214 DEBUG [RS:0;20b981ef5c27:40981 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f79f2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=20b981ef5c27/172.17.0.2:0 2024-11-24T20:45:43,214 DEBUG [RS:1;20b981ef5c27:40419 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ddd992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=20b981ef5c27/172.17.0.2:0 2024-11-24T20:45:43,215 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T20:45:43,215 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:43,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:43,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T20:45:43,225 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T20:45:43,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:43,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:43,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T20:45:43,230 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T20:45:43,231 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:43,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:43,231 DEBUG [RS:1;20b981ef5c27:40419 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;20b981ef5c27:40419 2024-11-24T20:45:43,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T20:45:43,234 DEBUG [RS:0;20b981ef5c27:40981 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;20b981ef5c27:40981 2024-11-24T20:45:43,235 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T20:45:43,235 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T20:45:43,235 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T20:45:43,235 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T20:45:43,236 DEBUG [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T20:45:43,236 DEBUG [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T20:45:43,237 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T20:45:43,237 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:43,237 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:43,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T20:45:43,238 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(2659): reportForDuty to master=20b981ef5c27,36507,1732481140790 with port=40981, startcode=1732481141837 2024-11-24T20:45:43,238 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(2659): reportForDuty to master=20b981ef5c27,36507,1732481140790 with port=40419, startcode=1732481142005 2024-11-24T20:45:43,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740 2024-11-24T20:45:43,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740 2024-11-24T20:45:43,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T20:45:43,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T20:45:43,240 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T20:45:43,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T20:45:43,249 DEBUG [RS:0;20b981ef5c27:40981 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T20:45:43,249 DEBUG [RS:1;20b981ef5c27:40419 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T20:45:43,252 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:45:43,253 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72663409, jitterRate=0.08276917040348053}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T20:45:43,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732481143210Initializing all the Stores at 1732481143210Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481143210Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481143210Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481143210Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481143210Cleaning up temporary data from old regions at 1732481143239 (+29 ms)Region opened successfully at 1732481143257 (+18 ms) 2024-11-24T20:45:43,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T20:45:43,257 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T20:45:43,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T20:45:43,258 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T20:45:43,258 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T20:45:43,260 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T20:45:43,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732481143257Disabling compacts and flushes for region at 1732481143257Disabling writes for close at 1732481143258 (+1 ms)Writing region close event to WAL at 1732481143260 (+2 ms)Closed at 1732481143260 2024-11-24T20:45:43,262 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T20:45:43,262 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T20:45:43,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T20:45:43,278 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T20:45:43,280 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T20:45:43,316 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55663, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T20:45:43,316 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57781, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T20:45:43,323 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36507 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 20b981ef5c27,40419,1732481142005 2024-11-24T20:45:43,326 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36507 {}] master.ServerManager(517): Registering regionserver=20b981ef5c27,40419,1732481142005 2024-11-24T20:45:43,336 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36507 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 20b981ef5c27,40981,1732481141837 2024-11-24T20:45:43,336 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36507 {}] master.ServerManager(517): Registering regionserver=20b981ef5c27,40981,1732481141837 2024-11-24T20:45:43,340 DEBUG [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 2024-11-24T20:45:43,340 DEBUG [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 2024-11-24T20:45:43,341 DEBUG [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T20:45:43,341 DEBUG [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T20:45:43,341 DEBUG [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T20:45:43,341 DEBUG [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T20:45:43,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T20:45:43,393 DEBUG [RS:1;20b981ef5c27:40419 {}] zookeeper.ZKUtil(111): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/20b981ef5c27,40419,1732481142005 2024-11-24T20:45:43,393 DEBUG [RS:0;20b981ef5c27:40981 {}] zookeeper.ZKUtil(111): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/20b981ef5c27,40981,1732481141837 2024-11-24T20:45:43,393 WARN [RS:0;20b981ef5c27:40981 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T20:45:43,393 WARN [RS:1;20b981ef5c27:40419 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T20:45:43,393 INFO [RS:1;20b981ef5c27:40419 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:45:43,393 INFO [RS:0;20b981ef5c27:40981 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:45:43,394 DEBUG [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,40419,1732481142005 2024-11-24T20:45:43,394 DEBUG [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,40981,1732481141837 2024-11-24T20:45:43,395 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [20b981ef5c27,40419,1732481142005] 2024-11-24T20:45:43,396 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [20b981ef5c27,40981,1732481141837] 2024-11-24T20:45:43,420 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T20:45:43,420 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T20:45:43,431 WARN [20b981ef5c27:36507 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T20:45:43,438 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T20:45:43,438 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T20:45:43,443 INFO [RS:1;20b981ef5c27:40419 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T20:45:43,443 INFO [RS:0;20b981ef5c27:40981 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T20:45:43,443 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,443 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,444 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T20:45:43,445 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T20:45:43,449 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T20:45:43,449 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T20:45:43,450 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,450 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,450 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,450 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/20b981ef5c27:0, corePoolSize=2, maxPoolSize=2 2024-11-24T20:45:43,451 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/20b981ef5c27:0, corePoolSize=2, maxPoolSize=2 2024-11-24T20:45:43,451 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,451 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,452 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,452 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,452 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,452 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,452 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,452 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,452 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,452 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:45:43,452 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,452 DEBUG [RS:1;20b981ef5c27:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:45:43,452 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:43,453 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:45:43,453 DEBUG [RS:0;20b981ef5c27:40981 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:45:43,454 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,454 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,454 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,454 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,454 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,454 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,454 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,40981,1732481141837-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T20:45:43,455 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,455 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,455 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,455 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,455 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,40419,1732481142005-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T20:45:43,476 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T20:45:43,477 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,40419,1732481142005-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,478 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,478 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.Replication(171): 20b981ef5c27,40419,1732481142005 started 2024-11-24T20:45:43,479 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T20:45:43,479 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,40981,1732481141837-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,479 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,479 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.Replication(171): 20b981ef5c27,40981,1732481141837 started 2024-11-24T20:45:43,495 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,496 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(1482): Serving as 20b981ef5c27,40419,1732481142005, RpcServer on 20b981ef5c27/172.17.0.2:40419, sessionid=0x1016e9842920002 2024-11-24T20:45:43,497 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T20:45:43,497 DEBUG [RS:1;20b981ef5c27:40419 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 20b981ef5c27,40419,1732481142005 2024-11-24T20:45:43,497 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,40419,1732481142005' 2024-11-24T20:45:43,497 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T20:45:43,498 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:43,498 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T20:45:43,498 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(1482): Serving as 20b981ef5c27,40981,1732481141837, RpcServer on 20b981ef5c27/172.17.0.2:40981, sessionid=0x1016e9842920001 2024-11-24T20:45:43,498 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T20:45:43,499 DEBUG [RS:0;20b981ef5c27:40981 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 20b981ef5c27,40981,1732481141837 2024-11-24T20:45:43,499 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T20:45:43,499 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T20:45:43,499 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,40981,1732481141837' 2024-11-24T20:45:43,499 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T20:45:43,499 DEBUG [RS:1;20b981ef5c27:40419 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 20b981ef5c27,40419,1732481142005 2024-11-24T20:45:43,499 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,40419,1732481142005' 2024-11-24T20:45:43,499 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T20:45:43,499 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T20:45:43,500 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T20:45:43,500 DEBUG [RS:1;20b981ef5c27:40419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T20:45:43,500 INFO [RS:1;20b981ef5c27:40419 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T20:45:43,500 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T20:45:43,500 INFO [RS:1;20b981ef5c27:40419 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T20:45:43,500 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T20:45:43,500 DEBUG [RS:0;20b981ef5c27:40981 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 20b981ef5c27,40981,1732481141837 2024-11-24T20:45:43,500 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,40981,1732481141837' 2024-11-24T20:45:43,501 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T20:45:43,501 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T20:45:43,502 DEBUG [RS:0;20b981ef5c27:40981 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T20:45:43,502 INFO [RS:0;20b981ef5c27:40981 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T20:45:43,502 INFO [RS:0;20b981ef5c27:40981 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T20:45:43,606 INFO [RS:0;20b981ef5c27:40981 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T20:45:43,606 INFO [RS:1;20b981ef5c27:40419 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T20:45:43,607 INFO [RS:1;20b981ef5c27:40419 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=20b981ef5c27%2C40419%2C1732481142005, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,40419,1732481142005, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs, maxLogs=55 2024-11-24T20:45:43,607 INFO [RS:0;20b981ef5c27:40981 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=20b981ef5c27%2C40981%2C1732481141837, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,40981,1732481141837, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs, maxLogs=55 2024-11-24T20:45:43,627 INFO [RS:1;20b981ef5c27:40419 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,40419,1732481142005/20b981ef5c27%2C40419%2C1732481142005.1732481143611 2024-11-24T20:45:43,627 INFO [RS:0;20b981ef5c27:40981 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,40981,1732481141837/20b981ef5c27%2C40981%2C1732481141837.1732481143611 2024-11-24T20:45:43,627 DEBUG [RS:0;20b981ef5c27:40981 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T20:45:43,627 DEBUG [RS:1;20b981ef5c27:40419 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T20:45:43,684 DEBUG [20b981ef5c27:36507 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T20:45:43,697 DEBUG [20b981ef5c27:36507 {}] balancer.BalancerClusterState(204): Hosts are {20b981ef5c27=0} racks are {/default-rack=0} 2024-11-24T20:45:43,699 DEBUG [20b981ef5c27:36507 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T20:45:43,700 DEBUG [20b981ef5c27:36507 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T20:45:43,700 DEBUG [20b981ef5c27:36507 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T20:45:43,700 DEBUG [20b981ef5c27:36507 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T20:45:43,700 INFO [20b981ef5c27:36507 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T20:45:43,700 INFO [20b981ef5c27:36507 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T20:45:43,700 DEBUG [20b981ef5c27:36507 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T20:45:43,707 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=20b981ef5c27,40419,1732481142005 2024-11-24T20:45:43,711 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 20b981ef5c27,40419,1732481142005, state=OPENING 2024-11-24T20:45:43,734 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T20:45:43,745 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:43,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:43,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:43,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:43,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:43,748 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:43,752 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T20:45:43,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=20b981ef5c27,40419,1732481142005}] 2024-11-24T20:45:43,930 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T20:45:43,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50925, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T20:45:43,941 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T20:45:43,941 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:45:43,942 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T20:45:43,944 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=20b981ef5c27%2C40419%2C1732481142005.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,40419,1732481142005, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs, maxLogs=55 2024-11-24T20:45:43,964 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,40419,1732481142005/20b981ef5c27%2C40419%2C1732481142005.meta.1732481143945.meta 2024-11-24T20:45:43,964 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T20:45:43,965 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:45:43,967 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T20:45:43,969 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T20:45:43,973 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T20:45:43,976 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T20:45:43,977 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:43,977 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T20:45:43,977 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T20:45:43,979 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T20:45:43,980 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T20:45:43,980 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:43,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:43,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T20:45:43,981 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T20:45:43,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:43,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:43,982 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T20:45:43,982 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T20:45:43,982 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:43,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:43,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T20:45:43,983 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T20:45:43,984 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:43,984 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:43,984 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T20:45:43,986 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740 2024-11-24T20:45:43,986 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740 2024-11-24T20:45:43,987 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T20:45:43,987 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T20:45:43,987 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T20:45:43,988 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T20:45:43,989 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70579173, jitterRate=0.051711633801460266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T20:45:43,989 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T20:45:43,991 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732481143978Writing region info on filesystem at 1732481143978Initializing all the Stores at 1732481143978Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481143978Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481143979 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481143979Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481143979Cleaning up temporary data from old regions at 1732481143987 (+8 ms)Running coprocessor post-open hooks at 1732481143989 (+2 ms)Region opened successfully at 1732481143991 (+2 ms) 2024-11-24T20:45:43,997 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732481143921 2024-11-24T20:45:44,007 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T20:45:44,007 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T20:45:44,008 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,40419,1732481142005 2024-11-24T20:45:44,010 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 20b981ef5c27,40419,1732481142005, state=OPEN 2024-11-24T20:45:44,061 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T20:45:44,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T20:45:44,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T20:45:44,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:44,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:44,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:44,063 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=20b981ef5c27,40419,1732481142005 2024-11-24T20:45:44,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T20:45:44,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=20b981ef5c27,40419,1732481142005 in 309 msec 2024-11-24T20:45:44,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T20:45:44,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 802 msec 2024-11-24T20:45:44,077 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T20:45:44,077 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T20:45:44,096 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:45:44,097 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,40419,1732481142005, seqNum=-1] 2024-11-24T20:45:44,114 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:45:44,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55437, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:45:44,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0310 sec 2024-11-24T20:45:44,135 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732481144135, completionTime=-1 2024-11-24T20:45:44,138 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T20:45:44,138 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T20:45:44,166 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T20:45:44,166 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732481204166 2024-11-24T20:45:44,166 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732481264166 2024-11-24T20:45:44,166 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-24T20:45:44,169 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T20:45:44,176 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,36507,1732481140790-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:44,176 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,36507,1732481140790-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:44,177 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,36507,1732481140790-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:44,178 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-20b981ef5c27:36507, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:44,179 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:44,179 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:44,185 DEBUG [master/20b981ef5c27:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T20:45:44,208 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.982sec 2024-11-24T20:45:44,210 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T20:45:44,212 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T20:45:44,213 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T20:45:44,214 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T20:45:44,214 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T20:45:44,214 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,36507,1732481140790-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T20:45:44,215 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,36507,1732481140790-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T20:45:44,216 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T20:45:44,217 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T20:45:44,218 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,36507,1732481140790-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:44,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6098c9c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:45:44,227 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T20:45:44,227 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T20:45:44,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,36507,-1 for getting cluster id 2024-11-24T20:45:44,234 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:45:44,242 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '93158d97-66e5-41c8-88a4-7d9ba5e3dd52' 2024-11-24T20:45:44,245 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:45:44,245 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "93158d97-66e5-41c8-88a4-7d9ba5e3dd52" 2024-11-24T20:45:44,246 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1b0342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:45:44,246 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,36507,-1] 2024-11-24T20:45:44,248 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:45:44,250 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:44,252 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:45:44,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ae5fc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:45:44,255 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:45:44,261 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,40419,1732481142005, seqNum=-1] 2024-11-24T20:45:44,261 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:45:44,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:45:44,296 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=20b981ef5c27,36507,1732481140790 2024-11-24T20:45:44,300 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T20:45:44,304 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 20b981ef5c27,36507,1732481140790 2024-11-24T20:45:44,306 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2134883d 2024-11-24T20:45:44,307 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T20:45:44,309 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T20:45:44,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732481144296', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:45:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732481144296 2024-11-24T20:45:44,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732481144296 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T20:45:44,326 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:44,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732481144296" procId is: 4 2024-11-24T20:45:44,327 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732481144296 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T20:45:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:45:44,340 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732481144296-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0a2dc06fa8ce1604c6dc49133a404c65, NAME => 'testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732481144296', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 2024-11-24T20:45:44,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732481144296-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:44,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732481144296-pool-0 {}] regionserver.HRegion(1722): Closing 0a2dc06fa8ce1604c6dc49133a404c65, disabling compactions & flushes 2024-11-24T20:45:44,348 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732481144296-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:44,348 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732481144296-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:44,348 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732481144296-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. after waiting 0 ms 2024-11-24T20:45:44,348 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732481144296-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:44,348 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732481144296-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:44,348 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732481144296-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0a2dc06fa8ce1604c6dc49133a404c65: Waiting for close lock at 1732481144347Disabling compacts and flushes for region at 1732481144347Disabling writes for close at 1732481144348 (+1 ms)Writing region close event to WAL at 1732481144348Closed at 1732481144348 2024-11-24T20:45:44,349 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732481144296 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T20:45:44,353 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732481144349"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732481144349"}]},"ts":"1732481144349"} 2024-11-24T20:45:44,358 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T20:45:44,359 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732481144296 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T20:45:44,362 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732481144296","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481144359"}]},"ts":"1732481144359"} 2024-11-24T20:45:44,366 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732481144296, state=ENABLING in hbase:meta 2024-11-24T20:45:44,366 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {20b981ef5c27=0} racks are {/default-rack=0} 2024-11-24T20:45:44,367 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T20:45:44,367 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T20:45:44,367 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T20:45:44,367 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T20:45:44,368 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T20:45:44,368 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T20:45:44,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T20:45:44,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, ASSIGN}] 2024-11-24T20:45:44,371 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, ASSIGN 2024-11-24T20:45:44,372 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, ASSIGN; state=OFFLINE, location=20b981ef5c27,40419,1732481142005; forceNewPlan=false, retain=false 2024-11-24T20:45:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:45:44,525 INFO [20b981ef5c27:36507 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T20:45:44,526 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0a2dc06fa8ce1604c6dc49133a404c65, regionState=OPENING, regionLocation=20b981ef5c27,40419,1732481142005 2024-11-24T20:45:44,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, ASSIGN because future has completed 2024-11-24T20:45:44,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a2dc06fa8ce1604c6dc49133a404c65, server=20b981ef5c27,40419,1732481142005}] 2024-11-24T20:45:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:45:44,692 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:44,693 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a2dc06fa8ce1604c6dc49133a404c65, NAME => 'testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:45:44,695 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732481144296 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,695 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:44,695 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,695 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,698 INFO [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,703 INFO [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a2dc06fa8ce1604c6dc49133a404c65 columnFamilyName family 2024-11-24T20:45:44,703 DEBUG [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:44,703 INFO [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] regionserver.HStore(327): Store=0a2dc06fa8ce1604c6dc49133a404c65/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:44,704 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,705 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,705 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,706 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,706 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,707 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,712 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:45:44,713 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0a2dc06fa8ce1604c6dc49133a404c65; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72825773, jitterRate=0.08518858253955841}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T20:45:44,713 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:44,715 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0a2dc06fa8ce1604c6dc49133a404c65: Running coprocessor pre-open hook at 1732481144697Writing region info on filesystem at 1732481144697Initializing all the Stores at 1732481144697Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481144697Cleaning up temporary data from old regions at 1732481144706 (+9 ms)Running coprocessor post-open hooks at 1732481144713 (+7 ms)Region opened successfully at 1732481144715 (+2 ms) 2024-11-24T20:45:44,716 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65., pid=6, masterSystemTime=1732481144686 2024-11-24T20:45:44,720 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:44,720 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:44,720 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0a2dc06fa8ce1604c6dc49133a404c65, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,40419,1732481142005 2024-11-24T20:45:44,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a2dc06fa8ce1604c6dc49133a404c65, server=20b981ef5c27,40419,1732481142005 because future has completed 2024-11-24T20:45:44,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T20:45:44,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a2dc06fa8ce1604c6dc49133a404c65, server=20b981ef5c27,40419,1732481142005 in 193 msec 2024-11-24T20:45:44,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T20:45:44,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, ASSIGN in 358 msec 2024-11-24T20:45:44,730 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732481144296 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T20:45:44,730 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732481144296","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481144730"}]},"ts":"1732481144730"} 2024-11-24T20:45:44,733 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732481144296, state=ENABLED in hbase:meta 2024-11-24T20:45:44,734 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732481144296 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T20:45:44,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732481144296 in 417 msec 2024-11-24T20:45:44,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:45:44,961 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732481144296 completed 2024-11-24T20:45:44,961 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732481144296 get assigned. Timeout = 60000ms 2024-11-24T20:45:44,965 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T20:45:44,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732481144296 assigned to meta. Checking AM states. 2024-11-24T20:45:44,974 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T20:45:44,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732481144296 assigned. 2024-11-24T20:45:44,984 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732481144296', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65., hostname=20b981ef5c27,40419,1732481142005, seqNum=2] 2024-11-24T20:45:44,995 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0a2dc06fa8ce1604c6dc49133a404c65 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T20:45:45,016 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b215f5d8178a483881994c2d87f4b8dd 2024-11-24T20:45:45,053 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/.tmp/family/b215f5d8178a483881994c2d87f4b8dd is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T20:45:45,058 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/.tmp/family/b215f5d8178a483881994c2d87f4b8dd 2024-11-24T20:45:45,086 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/.tmp/family/b215f5d8178a483881994c2d87f4b8dd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd 2024-11-24T20:45:45,088 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd, delay=929 ms 2024-11-24T20:45:45,089 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T20:45:45,096 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 0a2dc06fa8ce1604c6dc49133a404c65 in 99ms, sequenceid=6, compaction requested=false 2024-11-24T20:45:45,097 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732481144296' 2024-11-24T20:45:45,100 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0a2dc06fa8ce1604c6dc49133a404c65: 2024-11-24T20:45:46,022 DEBUG [hfile-prefetch-1732481145088 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b215f5d8178a483881994c2d87f4b8dd_0. 2024-11-24T20:45:46,029 DEBUG [hfile-prefetch-1732481145088 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b215f5d8178a483881994c2d87f4b8dd_117. 2024-11-24T20:45:46,030 DEBUG [hfile-prefetch-1732481145088 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd, with total blocks 2, and data blocks 1 2024-11-24T20:45:46,030 DEBUG [hfile-prefetch-1732481145088 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T20:45:46,030 DEBUG [hfile-prefetch-1732481145088 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T20:45:46,030 DEBUG [hfile-prefetch-1732481145088 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd. Will sleep 100ms and try the verification again. 2024-11-24T20:45:46,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=0a2dc06fa8ce1604c6dc49133a404c65, source=20b981ef5c27,40419,1732481142005, destination=20b981ef5c27,40981,1732481141837, warming up region on 20b981ef5c27,40981,1732481141837 2024-11-24T20:45:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T20:45:46,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=0a2dc06fa8ce1604c6dc49133a404c65, source=20b981ef5c27,40419,1732481142005, destination=20b981ef5c27,40981,1732481141837, running balancer 2024-11-24T20:45:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, REOPEN/MOVE 2024-11-24T20:45:46,121 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, REOPEN/MOVE 2024-11-24T20:45:46,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46769, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T20:45:46,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,123 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0a2dc06fa8ce1604c6dc49133a404c65, regionState=CLOSING, regionLocation=20b981ef5c27,40419,1732481142005 2024-11-24T20:45:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] regionserver.HRegion(7855): Warmup {ENCODED => 0a2dc06fa8ce1604c6dc49133a404c65, NAME => 'testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:45:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:46,124 INFO [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,125 INFO [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a2dc06fa8ce1604c6dc49133a404c65 columnFamilyName family 2024-11-24T20:45:46,125 DEBUG [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:46,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, REOPEN/MOVE because future has completed 2024-11-24T20:45:46,128 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-24T20:45:46,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0a2dc06fa8ce1604c6dc49133a404c65, server=20b981ef5c27,40419,1732481142005}] 2024-11-24T20:45:46,129 DEBUG [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd 2024-11-24T20:45:46,129 INFO [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] regionserver.HStore(327): Store=0a2dc06fa8ce1604c6dc49133a404c65/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] regionserver.HRegion(1722): Closing 0a2dc06fa8ce1604c6dc49133a404c65, disabling compactions & flushes 2024-11-24T20:45:46,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. after waiting 0 ms 2024-11-24T20:45:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,130 DEBUG [hfile-prefetch-1732481145088 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd, with total blocks 2, and data blocks 1 2024-11-24T20:45:46,130 DEBUG [hfile-prefetch-1732481145088 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T20:45:46,130 DEBUG [hfile-prefetch-1732481145088 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T20:45:46,131 DEBUG [hfile-prefetch-1732481145088 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd. Will sleep 100ms and try the verification again. 2024-11-24T20:45:46,131 DEBUG [StoreFileCloser-0a2dc06fa8ce1604c6dc49133a404c65-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd 2024-11-24T20:45:46,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] regionserver.HRegion(1676): Region close journal for 0a2dc06fa8ce1604c6dc49133a404c65: Waiting for close lock at 1732481146130Disabling compacts and flushes for region at 1732481146130Disabling writes for close at 1732481146130Writing region close event to WAL at 1732481146131 (+1 ms)Closed at 1732481146131 2024-11-24T20:45:46,133 WARN [hfile-prefetch-1732481145088 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 10 more 2024-11-24T20:45:46,134 DEBUG [hfile-prefetch-1732481145088 {}] hfile.PrefetchExecutor(121): Prefetch completed for b215f5d8178a483881994c2d87f4b8dd 2024-11-24T20:45:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36507 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T20:45:46,198 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T20:45:46,198 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T20:45:46,198 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T20:45:46,198 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T20:45:46,219 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T20:45:46,219 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732481146198 2024-11-24T20:45:46,219 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T20:45:46,287 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,287 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-24T20:45:46,287 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 0a2dc06fa8ce1604c6dc49133a404c65, disabling compactions & flushes 2024-11-24T20:45:46,288 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,288 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,288 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. after waiting 0 ms 2024-11-24T20:45:46,288 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,289 DEBUG [StoreFileCloser-0a2dc06fa8ce1604c6dc49133a404c65-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file b215f5d8178a483881994c2d87f4b8dd, starting offset: 0, end offset: 9223372036854775807 2024-11-24T20:45:46,290 DEBUG [StoreFileCloser-0a2dc06fa8ce1604c6dc49133a404c65-family-1 {}] bucket.BucketCache(856): removed key b215f5d8178a483881994c2d87f4b8dd_0 from back map with offset lock 2094080 in the evict process 2024-11-24T20:45:46,295 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T20:45:46,296 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,297 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 0a2dc06fa8ce1604c6dc49133a404c65: Waiting for close lock at 1732481146287Running coprocessor pre-close hooks at 1732481146287Disabling compacts and flushes for region at 1732481146287Disabling writes for close at 1732481146288 (+1 ms)Writing region close event to WAL at 1732481146291 (+3 ms)Running coprocessor post-close hooks at 1732481146295 (+4 ms)Closed at 1732481146296 (+1 ms) 2024-11-24T20:45:46,297 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 0a2dc06fa8ce1604c6dc49133a404c65 move to 20b981ef5c27,40981,1732481141837 record at close sequenceid=6 2024-11-24T20:45:46,300 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,301 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0a2dc06fa8ce1604c6dc49133a404c65, regionState=CLOSED 2024-11-24T20:45:46,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0a2dc06fa8ce1604c6dc49133a404c65, server=20b981ef5c27,40419,1732481142005 because future has completed 2024-11-24T20:45:46,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T20:45:46,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 0a2dc06fa8ce1604c6dc49133a404c65, server=20b981ef5c27,40419,1732481142005 in 177 msec 2024-11-24T20:45:46,309 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, REOPEN/MOVE; state=CLOSED, location=20b981ef5c27,40981,1732481141837; forceNewPlan=false, retain=false 2024-11-24T20:45:46,319 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T20:45:46,319 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T20:45:46,319 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T20:45:46,319 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T20:45:46,323 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T20:45:46,323 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732481146319 2024-11-24T20:45:46,323 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T20:45:46,343 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T20:45:46,343 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T20:45:46,447 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T20:45:46,447 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T20:45:46,450 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T20:45:46,450 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732481146343 2024-11-24T20:45:46,450 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T20:45:46,459 INFO [20b981ef5c27:36507 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T20:45:46,460 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0a2dc06fa8ce1604c6dc49133a404c65, regionState=OPENING, regionLocation=20b981ef5c27,40981,1732481141837 2024-11-24T20:45:46,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, REOPEN/MOVE because future has completed 2024-11-24T20:45:46,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a2dc06fa8ce1604c6dc49133a404c65, server=20b981ef5c27,40981,1732481141837}] 2024-11-24T20:45:46,620 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,621 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a2dc06fa8ce1604c6dc49133a404c65, NAME => 'testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:45:46,622 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732481144296 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,622 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:46,622 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,623 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,625 INFO [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,626 INFO [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a2dc06fa8ce1604c6dc49133a404c65 columnFamilyName family 2024-11-24T20:45:46,627 DEBUG [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:46,630 DEBUG [StoreFileOpener-0a2dc06fa8ce1604c6dc49133a404c65-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd, delay=944 ms 2024-11-24T20:45:46,632 DEBUG [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd 2024-11-24T20:45:46,633 INFO [StoreOpener-0a2dc06fa8ce1604c6dc49133a404c65-1 {}] regionserver.HStore(327): Store=0a2dc06fa8ce1604c6dc49133a404c65/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:46,633 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,634 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,634 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,635 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,635 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,640 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,640 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 0a2dc06fa8ce1604c6dc49133a404c65; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74979966, jitterRate=0.11728855967521667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T20:45:46,641 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,641 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 0a2dc06fa8ce1604c6dc49133a404c65: Running coprocessor pre-open hook at 1732481146623Writing region info on filesystem at 1732481146623Initializing all the Stores at 1732481146624 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481146624Cleaning up temporary data from old regions at 1732481146635 (+11 ms)Running coprocessor post-open hooks at 1732481146641 (+6 ms)Region opened successfully at 1732481146641 2024-11-24T20:45:46,642 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65., pid=9, masterSystemTime=1732481146617 2024-11-24T20:45:46,645 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,645 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,645 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0a2dc06fa8ce1604c6dc49133a404c65, regionState=OPEN, openSeqNum=10, regionLocation=20b981ef5c27,40981,1732481141837 2024-11-24T20:45:46,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a2dc06fa8ce1604c6dc49133a404c65, server=20b981ef5c27,40981,1732481141837 because future has completed 2024-11-24T20:45:46,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T20:45:46,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a2dc06fa8ce1604c6dc49133a404c65, server=20b981ef5c27,40981,1732481141837 in 185 msec 2024-11-24T20:45:46,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732481144296, region=0a2dc06fa8ce1604c6dc49133a404c65, REOPEN/MOVE in 531 msec 2024-11-24T20:45:46,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T20:45:46,675 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:45:46,675 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:45:46,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:46,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:46,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:45:46,679 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T20:45:46,679 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-24T20:45:46,679 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=20b981ef5c27,36507,1732481140790 2024-11-24T20:45:46,692 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:46,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:46,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:46,692 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:46,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:46,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:46,692 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T20:45:46,693 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:46,693 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:45:46,693 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:46,693 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:46,693 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:45:46,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:46,695 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '20b981ef5c27,40981,1732481141837' ***** 2024-11-24T20:45:46,695 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T20:45:46,695 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '20b981ef5c27,40419,1732481142005' ***** 2024-11-24T20:45:46,695 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T20:45:46,696 INFO [RS:1;20b981ef5c27:40419 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T20:45:46,696 INFO [RS:0;20b981ef5c27:40981 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T20:45:46,696 INFO [RS:1;20b981ef5c27:40419 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/bucket.persistence 2024-11-24T20:45:46,696 INFO [RS:0;20b981ef5c27:40981 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/bucket.persistence 2024-11-24T20:45:46,696 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T20:45:46,696 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T20:45:46,696 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T20:45:46,696 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T20:45:46,696 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T20:45:46,697 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T20:45:46,697 INFO [RS:1;20b981ef5c27:40419 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T20:45:46,697 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T20:45:46,697 INFO [RS:0;20b981ef5c27:40981 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T20:45:46,697 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T20:45:46,707 DEBUG [RS:1;20b981ef5c27:40419 {}] bucket.BucketCache(1487): Thread RS:1;20b981ef5c27:40419 started persisting bucket cache to file 2024-11-24T20:45:46,707 DEBUG [RS:1;20b981ef5c27:40419 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T20:45:46,707 DEBUG [RS:1;20b981ef5c27:40419 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T20:45:46,707 DEBUG [RS:0;20b981ef5c27:40981 {}] bucket.BucketCache(1487): Thread RS:0;20b981ef5c27:40981 started persisting bucket cache to file 2024-11-24T20:45:46,708 DEBUG [RS:0;20b981ef5c27:40981 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T20:45:46,708 DEBUG [RS:0;20b981ef5c27:40981 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T20:45:46,710 DEBUG [RS:1;20b981ef5c27:40419 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T20:45:46,710 DEBUG [RS:1;20b981ef5c27:40419 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732481146707 2024-11-24T20:45:46,710 DEBUG [RS:1;20b981ef5c27:40419 {}] bucket.BucketCache(1509): Thread RS:1;20b981ef5c27:40419 finished persisting bucket cache to file, renaming 2024-11-24T20:45:46,710 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T20:45:46,710 ERROR [RS:0;20b981ef5c27:40981 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1056) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:45:46,711 DEBUG [RS:0;20b981ef5c27:40981 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T20:45:46,711 INFO [RS:1;20b981ef5c27:40419 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T20:45:46,711 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T20:45:46,711 DEBUG [RS:0;20b981ef5c27:40981 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732481146707 2024-11-24T20:45:46,711 DEBUG [RS:0;20b981ef5c27:40981 {}] bucket.BucketCache(1509): Thread RS:0;20b981ef5c27:40981 finished persisting bucket cache to file, renaming 2024-11-24T20:45:46,711 INFO [RS:1;20b981ef5c27:40419 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T20:45:46,711 WARN [RS:0;20b981ef5c27:40981 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-24T20:45:46,711 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T20:45:46,711 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(959): stopping server 20b981ef5c27,40419,1732481142005 2024-11-24T20:45:46,711 INFO [RS:1;20b981ef5c27:40419 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T20:45:46,711 INFO [RS:0;20b981ef5c27:40981 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T20:45:46,711 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T20:45:46,711 INFO [RS:0;20b981ef5c27:40981 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T20:45:46,711 INFO [RS:1;20b981ef5c27:40419 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;20b981ef5c27:40419. 2024-11-24T20:45:46,711 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(3091): Received CLOSE for 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,711 DEBUG [RS:1;20b981ef5c27:40419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:45:46,711 DEBUG [RS:1;20b981ef5c27:40419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:46,711 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T20:45:46,711 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T20:45:46,711 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T20:45:46,711 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T20:45:46,712 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(959): stopping server 20b981ef5c27,40981,1732481141837 2024-11-24T20:45:46,712 INFO [RS:0;20b981ef5c27:40981 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T20:45:46,712 INFO [RS:0;20b981ef5c27:40981 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;20b981ef5c27:40981. 2024-11-24T20:45:46,712 DEBUG [RS:0;20b981ef5c27:40981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:45:46,712 DEBUG [RS:0;20b981ef5c27:40981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:46,712 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T20:45:46,712 DEBUG [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T20:45:46,712 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T20:45:46,712 DEBUG [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(1325): Online Regions={0a2dc06fa8ce1604c6dc49133a404c65=testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65.} 2024-11-24T20:45:46,712 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T20:45:46,712 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T20:45:46,712 DEBUG [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(1351): Waiting on 0a2dc06fa8ce1604c6dc49133a404c65 2024-11-24T20:45:46,712 DEBUG [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T20:45:46,712 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0a2dc06fa8ce1604c6dc49133a404c65, disabling compactions & flushes 2024-11-24T20:45:46,712 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,712 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T20:45:46,713 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,713 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T20:45:46,713 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. after waiting 0 ms 2024-11-24T20:45:46,713 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,713 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T20:45:46,713 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T20:45:46,713 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T20:45:46,713 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-24T20:45:46,713 DEBUG [StoreFileCloser-0a2dc06fa8ce1604c6dc49133a404c65-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/family/b215f5d8178a483881994c2d87f4b8dd 2024-11-24T20:45:46,718 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnRegionMove1732481144296/0a2dc06fa8ce1604c6dc49133a404c65/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T20:45:46,718 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,718 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0a2dc06fa8ce1604c6dc49133a404c65: Waiting for close lock at 1732481146712Running coprocessor pre-close hooks at 1732481146712Disabling compacts and flushes for region at 1732481146712Disabling writes for close at 1732481146713 (+1 ms)Writing region close event to WAL at 1732481146714 (+1 ms)Running coprocessor post-close hooks at 1732481146718 (+4 ms)Closed at 1732481146718 2024-11-24T20:45:46,718 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65. 2024-11-24T20:45:46,722 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c4c8ae43244c491fafdb006a3c1acd53 2024-11-24T20:45:46,739 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/info/c4c8ae43244c491fafdb006a3c1acd53 is 205, key is testBlockEvictionOnRegionMove1732481144296,,1732481144310.0a2dc06fa8ce1604c6dc49133a404c65./info:regioninfo/1732481146645/Put/seqid=0 2024-11-24T20:45:46,740 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/info/c4c8ae43244c491fafdb006a3c1acd53 2024-11-24T20:45:46,748 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8673c937195e431c9a7db613cedba20b 2024-11-24T20:45:46,759 INFO [regionserver/20b981ef5c27:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:45:46,759 INFO [regionserver/20b981ef5c27:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:45:46,760 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/ns/8673c937195e431c9a7db613cedba20b is 43, key is default/ns:d/1732481144120/Put/seqid=0 2024-11-24T20:45:46,761 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/ns/8673c937195e431c9a7db613cedba20b 2024-11-24T20:45:46,769 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 813e9d9e8b8d4b80b54aa1d847eed423 2024-11-24T20:45:46,781 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/table/813e9d9e8b8d4b80b54aa1d847eed423 is 78, key is testBlockEvictionOnRegionMove1732481144296/table:state/1732481144730/Put/seqid=0 2024-11-24T20:45:46,781 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/table/813e9d9e8b8d4b80b54aa1d847eed423 2024-11-24T20:45:46,784 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/info/c4c8ae43244c491fafdb006a3c1acd53 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/info/c4c8ae43244c491fafdb006a3c1acd53 2024-11-24T20:45:46,785 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/info/c4c8ae43244c491fafdb006a3c1acd53, delay=933 ms 2024-11-24T20:45:46,786 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/info/c4c8ae43244c491fafdb006a3c1acd53, entries=15, sequenceid=15, filesize=7.6 K 2024-11-24T20:45:46,787 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/ns/8673c937195e431c9a7db613cedba20b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/ns/8673c937195e431c9a7db613cedba20b 2024-11-24T20:45:46,788 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/ns/8673c937195e431c9a7db613cedba20b, delay=979 ms 2024-11-24T20:45:46,789 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/ns/8673c937195e431c9a7db613cedba20b, entries=2, sequenceid=15, filesize=5.0 K 2024-11-24T20:45:46,789 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/table/813e9d9e8b8d4b80b54aa1d847eed423 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/table/813e9d9e8b8d4b80b54aa1d847eed423 2024-11-24T20:45:46,791 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/table/813e9d9e8b8d4b80b54aa1d847eed423, delay=955 ms 2024-11-24T20:45:46,792 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/table/813e9d9e8b8d4b80b54aa1d847eed423, entries=2, sequenceid=15, filesize=5.3 K 2024-11-24T20:45:46,792 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=15, compaction requested=false 2024-11-24T20:45:46,792 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T20:45:46,794 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/info/c4c8ae43244c491fafdb006a3c1acd53 2024-11-24T20:45:46,794 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/ns/8673c937195e431c9a7db613cedba20b 2024-11-24T20:45:46,795 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/table/813e9d9e8b8d4b80b54aa1d847eed423 2024-11-24T20:45:46,799 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T20:45:46,799 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T20:45:46,800 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T20:45:46,800 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732481146712Running coprocessor pre-close hooks at 1732481146712Disabling compacts and flushes for region at 1732481146712Disabling writes for close at 1732481146713 (+1 ms)Obtaining lock to block concurrent updates at 1732481146713Preparing flush snapshotting stores in 1588230740 at 1732481146713Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732481146714 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732481146714Flushing 1588230740/info: creating writer at 1732481146714Flushing 1588230740/info: appending metadata at 1732481146738 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732481146738Flushing 1588230740/ns: creating writer at 1732481146742 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732481146759 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732481146759Flushing 1588230740/table: creating writer at 1732481146763 (+4 ms)Flushing 1588230740/table: appending metadata at 1732481146780 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732481146780Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@779a3930: reopening flushed file at 1732481146784 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@264c113: reopening flushed file at 1732481146786 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24a5f1f: reopening flushed file at 1732481146789 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=15, compaction requested=false at 1732481146792 (+3 ms)Writing region close event to WAL at 1732481146795 (+3 ms)Running coprocessor post-close hooks at 1732481146799 (+4 ms)Closed at 1732481146800 (+1 ms) 2024-11-24T20:45:46,800 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T20:45:46,913 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(976): stopping server 20b981ef5c27,40419,1732481142005; all regions closed. 2024-11-24T20:45:46,913 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(976): stopping server 20b981ef5c27,40981,1732481141837; all regions closed. 2024-11-24T20:45:46,919 DEBUG [RS:1;20b981ef5c27:40419 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs 2024-11-24T20:45:46,919 DEBUG [RS:0;20b981ef5c27:40981 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs 2024-11-24T20:45:46,919 INFO [RS:1;20b981ef5c27:40419 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 20b981ef5c27%2C40419%2C1732481142005.meta:.meta(num 1732481143945) 2024-11-24T20:45:46,919 INFO [RS:0;20b981ef5c27:40981 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 20b981ef5c27%2C40981%2C1732481141837:(num 1732481143611) 2024-11-24T20:45:46,919 DEBUG [RS:0;20b981ef5c27:40981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:46,919 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:45:46,920 INFO [RS:0;20b981ef5c27:40981 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T20:45:46,920 INFO [RS:0;20b981ef5c27:40981 {}] hbase.ChoreService(370): Chore service for: regionserver/20b981ef5c27:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T20:45:46,921 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T20:45:46,921 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T20:45:46,921 INFO [regionserver/20b981ef5c27:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T20:45:46,921 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T20:45:46,921 INFO [RS:0;20b981ef5c27:40981 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T20:45:46,922 INFO [RS:0;20b981ef5c27:40981 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40981 2024-11-24T20:45:46,922 DEBUG [RS:1;20b981ef5c27:40419 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs 2024-11-24T20:45:46,922 INFO [RS:1;20b981ef5c27:40419 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 20b981ef5c27%2C40419%2C1732481142005:(num 1732481143611) 2024-11-24T20:45:46,922 DEBUG [RS:1;20b981ef5c27:40419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:46,922 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:45:46,922 INFO [RS:1;20b981ef5c27:40419 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T20:45:46,923 INFO [RS:1;20b981ef5c27:40419 {}] hbase.ChoreService(370): Chore service for: regionserver/20b981ef5c27:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T20:45:46,923 INFO [RS:1;20b981ef5c27:40419 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T20:45:46,923 INFO [regionserver/20b981ef5c27:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T20:45:46,924 INFO [RS:1;20b981ef5c27:40419 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40419 2024-11-24T20:45:46,955 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/20b981ef5c27,40419,1732481142005 2024-11-24T20:45:46,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T20:45:46,955 INFO [RS:1;20b981ef5c27:40419 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T20:45:46,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/20b981ef5c27,40981,1732481141837 2024-11-24T20:45:46,965 INFO [RS:0;20b981ef5c27:40981 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T20:45:46,966 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [20b981ef5c27,40419,1732481142005] 2024-11-24T20:45:46,986 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/20b981ef5c27,40419,1732481142005 already deleted, retry=false 2024-11-24T20:45:46,987 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 20b981ef5c27,40419,1732481142005 expired; onlineServers=1 2024-11-24T20:45:46,987 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [20b981ef5c27,40981,1732481141837] 2024-11-24T20:45:46,997 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/20b981ef5c27,40981,1732481141837 already deleted, retry=false 2024-11-24T20:45:46,997 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 20b981ef5c27,40981,1732481141837 expired; onlineServers=0 2024-11-24T20:45:46,997 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '20b981ef5c27,36507,1732481140790' ***** 2024-11-24T20:45:46,997 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T20:45:46,997 INFO [M:0;20b981ef5c27:36507 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T20:45:46,998 INFO [M:0;20b981ef5c27:36507 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T20:45:46,998 DEBUG [M:0;20b981ef5c27:36507 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T20:45:46,998 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T20:45:46,998 DEBUG [M:0;20b981ef5c27:36507 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T20:45:46,998 DEBUG [master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.large.0-1732481143194 {}] cleaner.HFileCleaner(306): Exit Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.large.0-1732481143194,5,FailOnTimeoutGroup] 2024-11-24T20:45:46,998 DEBUG [master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.small.0-1732481143195 {}] cleaner.HFileCleaner(306): Exit Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.small.0-1732481143195,5,FailOnTimeoutGroup] 2024-11-24T20:45:46,998 INFO [M:0;20b981ef5c27:36507 {}] hbase.ChoreService(370): Chore service for: master/20b981ef5c27:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T20:45:46,998 INFO [M:0;20b981ef5c27:36507 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T20:45:46,999 DEBUG [M:0;20b981ef5c27:36507 {}] master.HMaster(1795): Stopping service threads 2024-11-24T20:45:46,999 INFO [M:0;20b981ef5c27:36507 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T20:45:46,999 INFO [M:0;20b981ef5c27:36507 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T20:45:46,999 INFO [M:0;20b981ef5c27:36507 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T20:45:46,999 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T20:45:47,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T20:45:47,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,008 DEBUG [M:0;20b981ef5c27:36507 {}] zookeeper.ZKUtil(347): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T20:45:47,008 WARN [M:0;20b981ef5c27:36507 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T20:45:47,008 INFO [M:0;20b981ef5c27:36507 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/.lastflushedseqids 2024-11-24T20:45:47,016 INFO [M:0;20b981ef5c27:36507 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T20:45:47,016 INFO [M:0;20b981ef5c27:36507 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T20:45:47,016 DEBUG [M:0;20b981ef5c27:36507 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T20:45:47,016 INFO [M:0;20b981ef5c27:36507 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:47,016 DEBUG [M:0;20b981ef5c27:36507 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:47,016 DEBUG [M:0;20b981ef5c27:36507 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T20:45:47,016 DEBUG [M:0;20b981ef5c27:36507 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:47,016 INFO [M:0;20b981ef5c27:36507 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-24T20:45:47,025 DEBUG [M:0;20b981ef5c27:36507 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 205521cb350d42cdb692a30c9c905b1c 2024-11-24T20:45:47,038 DEBUG [M:0;20b981ef5c27:36507 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/205521cb350d42cdb692a30c9c905b1c is 82, key is hbase:meta,,1/info:regioninfo/1732481144008/Put/seqid=0 2024-11-24T20:45:47,039 INFO [M:0;20b981ef5c27:36507 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/205521cb350d42cdb692a30c9c905b1c 2024-11-24T20:45:47,046 DEBUG [M:0;20b981ef5c27:36507 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7e3ba5e7c94c413599d3f83b34045a05 2024-11-24T20:45:47,059 DEBUG [M:0;20b981ef5c27:36507 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e3ba5e7c94c413599d3f83b34045a05 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732481144735/Put/seqid=0 2024-11-24T20:45:47,059 INFO [M:0;20b981ef5c27:36507 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e3ba5e7c94c413599d3f83b34045a05 2024-11-24T20:45:47,066 DEBUG [M:0;20b981ef5c27:36507 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7546c91e157a49eea2d58df57f9dcf26 2024-11-24T20:45:47,076 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:47,076 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016e9842920002, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:47,077 INFO [RS:1;20b981ef5c27:40419 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T20:45:47,077 INFO [RS:1;20b981ef5c27:40419 {}] regionserver.HRegionServer(1031): Exiting; stopping=20b981ef5c27,40419,1732481142005; zookeeper connection closed. 2024-11-24T20:45:47,077 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@be3a6ba {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@be3a6ba 2024-11-24T20:45:47,078 DEBUG [M:0;20b981ef5c27:36507 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7546c91e157a49eea2d58df57f9dcf26 is 69, key is 20b981ef5c27,40419,1732481142005/rs:state/1732481143328/Put/seqid=0 2024-11-24T20:45:47,079 INFO [M:0;20b981ef5c27:36507 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7546c91e157a49eea2d58df57f9dcf26 2024-11-24T20:45:47,081 DEBUG [M:0;20b981ef5c27:36507 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/205521cb350d42cdb692a30c9c905b1c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/205521cb350d42cdb692a30c9c905b1c 2024-11-24T20:45:47,082 INFO [M:0;20b981ef5c27:36507 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/205521cb350d42cdb692a30c9c905b1c, entries=8, sequenceid=84, filesize=5.5 K 2024-11-24T20:45:47,082 DEBUG [M:0;20b981ef5c27:36507 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e3ba5e7c94c413599d3f83b34045a05 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7e3ba5e7c94c413599d3f83b34045a05 2024-11-24T20:45:47,084 INFO [M:0;20b981ef5c27:36507 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7e3ba5e7c94c413599d3f83b34045a05, entries=9, sequenceid=84, filesize=6.5 K 2024-11-24T20:45:47,084 DEBUG [M:0;20b981ef5c27:36507 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7546c91e157a49eea2d58df57f9dcf26 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7546c91e157a49eea2d58df57f9dcf26 2024-11-24T20:45:47,085 INFO [M:0;20b981ef5c27:36507 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7546c91e157a49eea2d58df57f9dcf26, entries=2, sequenceid=84, filesize=5.1 K 2024-11-24T20:45:47,086 INFO [M:0;20b981ef5c27:36507 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=84, compaction requested=false 2024-11-24T20:45:47,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:47,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40981-0x1016e9842920001, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:47,087 INFO [RS:0;20b981ef5c27:40981 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T20:45:47,087 INFO [RS:0;20b981ef5c27:40981 {}] regionserver.HRegionServer(1031): Exiting; stopping=20b981ef5c27,40981,1732481141837; zookeeper connection closed. 2024-11-24T20:45:47,087 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ab4bd03 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ab4bd03 2024-11-24T20:45:47,087 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-24T20:45:47,088 INFO [M:0;20b981ef5c27:36507 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:47,088 DEBUG [M:0;20b981ef5c27:36507 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732481147016Disabling compacts and flushes for region at 1732481147016Disabling writes for close at 1732481147016Obtaining lock to block concurrent updates at 1732481147017 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732481147017Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1732481147017Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732481147017Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732481147017Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732481147037 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732481147037Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732481147040 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732481147058 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732481147058Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732481147061 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732481147078 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732481147078Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2ee87e: reopening flushed file at 1732481147080 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a80c17b: reopening flushed file at 1732481147082 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c24cb7d: reopening flushed file at 1732481147084 (+2 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=84, compaction requested=false at 1732481147086 (+2 ms)Writing region close event to WAL at 1732481147088 (+2 ms)Closed at 1732481147088 2024-11-24T20:45:47,090 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T20:45:47,090 INFO [M:0;20b981ef5c27:36507 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T20:45:47,090 INFO [M:0;20b981ef5c27:36507 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36507 2024-11-24T20:45:47,090 INFO [M:0;20b981ef5c27:36507 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T20:45:47,202 INFO [M:0;20b981ef5c27:36507 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T20:45:47,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:47,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36507-0x1016e9842920000, quorum=127.0.0.1:60770, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:47,210 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T20:45:47,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T20:45:47,232 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:47,239 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: master/20b981ef5c27:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1732481146785 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/20b981ef5c27:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/20b981ef5c27:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732481146788 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732481146630 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/20b981ef5c27:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732481145088 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=556 (was 552) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11201 (was 11719) 2024-11-24T20:45:47,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=556, ProcessCount=11, AvailableMemoryMB=11200 2024-11-24T20:45:47,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/hadoop-log-dir so I do NOT create it in target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b 2024-11-24T20:45:47,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/hadoop-tmp-dir so I do NOT create it in target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b 2024-11-24T20:45:47,245 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:47,247 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/cluster_9a9b6429-a04a-6a7c-2698-de3c3548f675, deleteOnExit=true 2024-11-24T20:45:47,250 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/cluster_9a9b6429-a04a-6a7c-2698-de3c3548f675/zookeeper_0, clientPort=58231, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/cluster_9a9b6429-a04a-6a7c-2698-de3c3548f675/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/cluster_9a9b6429-a04a-6a7c-2698-de3c3548f675/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T20:45:47,251 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58231 2024-11-24T20:45:47,260 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 with version=8 2024-11-24T20:45:47,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/hbase-staging 2024-11-24T20:45:47,262 INFO [Time-limited test {}] client.ConnectionUtils(128): master/20b981ef5c27:0 server-side Connection retries=45 2024-11-24T20:45:47,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:47,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:47,263 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T20:45:47,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:47,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T20:45:47,263 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T20:45:47,263 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T20:45:47,264 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44901 2024-11-24T20:45:47,266 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44901 connecting to ZooKeeper ensemble=127.0.0.1:58231 2024-11-24T20:45:47,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:449010x0, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T20:45:47,312 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44901-0x1016e985d1d0000 connected 2024-11-24T20:45:47,393 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:47,393 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:47,394 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:47,395 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99, hbase.cluster.distributed=false 2024-11-24T20:45:47,397 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T20:45:47,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44901 2024-11-24T20:45:47,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44901 2024-11-24T20:45:47,405 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44901 2024-11-24T20:45:47,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44901 2024-11-24T20:45:47,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44901 2024-11-24T20:45:47,432 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/20b981ef5c27:0 server-side Connection retries=45 2024-11-24T20:45:47,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:47,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:47,433 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T20:45:47,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:47,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T20:45:47,433 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T20:45:47,434 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T20:45:47,434 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43263 2024-11-24T20:45:47,437 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43263 connecting to ZooKeeper ensemble=127.0.0.1:58231 2024-11-24T20:45:47,437 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:47,438 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:47,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432630x0, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T20:45:47,446 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:432630x0, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:47,447 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T20:45:47,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43263-0x1016e985d1d0001 connected 2024-11-24T20:45:47,449 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.cache 2024-11-24T20:45:47,449 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T20:45:47,449 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T20:45:47,449 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T20:45:47,449 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T20:45:47,449 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T20:45:47,450 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T20:45:47,450 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.persistence is complete. 2024-11-24T20:45:47,452 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T20:45:47,453 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T20:45:47,454 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T20:45:47,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T20:45:47,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43263 2024-11-24T20:45:47,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43263 2024-11-24T20:45:47,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43263 2024-11-24T20:45:47,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43263 2024-11-24T20:45:47,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43263 2024-11-24T20:45:47,481 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/20b981ef5c27:0 server-side Connection retries=45 2024-11-24T20:45:47,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:47,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:47,482 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T20:45:47,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:47,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T20:45:47,482 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T20:45:47,482 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T20:45:47,484 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33461 2024-11-24T20:45:47,487 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33461 connecting to ZooKeeper ensemble=127.0.0.1:58231 2024-11-24T20:45:47,487 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:47,487 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:47,498 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334610x0, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T20:45:47,499 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33461-0x1016e985d1d0002 connected 2024-11-24T20:45:47,499 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:47,499 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T20:45:47,500 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.cache 2024-11-24T20:45:47,500 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T20:45:47,500 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T20:45:47,500 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T20:45:47,504 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T20:45:47,504 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T20:45:47,504 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T20:45:47,505 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T20:45:47,505 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T20:45:47,505 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.persistence is complete. 2024-11-24T20:45:47,506 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T20:45:47,507 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T20:45:47,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33461 2024-11-24T20:45:47,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33461 2024-11-24T20:45:47,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33461 2024-11-24T20:45:47,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33461 2024-11-24T20:45:47,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33461 2024-11-24T20:45:47,526 DEBUG [M:0;20b981ef5c27:44901 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;20b981ef5c27:44901 2024-11-24T20:45:47,526 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/20b981ef5c27,44901,1732481147262 2024-11-24T20:45:47,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:47,538 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:47,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:47,538 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/20b981ef5c27,44901,1732481147262 2024-11-24T20:45:47,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,551 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T20:45:47,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T20:45:47,551 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,551 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T20:45:47,552 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/20b981ef5c27,44901,1732481147262 from backup master directory 2024-11-24T20:45:47,561 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:47,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:47,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/20b981ef5c27,44901,1732481147262 2024-11-24T20:45:47,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:45:47,561 WARN [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T20:45:47,561 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=20b981ef5c27,44901,1732481147262 2024-11-24T20:45:47,562 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/hbase.id] with ID: a868b0ac-f266-43e4-bc01-b920220e725d 2024-11-24T20:45:47,563 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/.tmp/hbase.id 2024-11-24T20:45:47,565 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/hbase.id] 2024-11-24T20:45:47,596 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:47,596 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T20:45:47,599 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-24T20:45:47,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,614 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,620 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:45:47,622 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T20:45:47,623 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:45:47,644 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store 2024-11-24T20:45:47,652 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:47,652 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T20:45:47,652 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:47,652 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:47,652 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T20:45:47,652 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:47,652 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:47,652 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732481147652Disabling compacts and flushes for region at 1732481147652Disabling writes for close at 1732481147652Writing region close event to WAL at 1732481147652Closed at 1732481147652 2024-11-24T20:45:47,653 WARN [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/.initializing 2024-11-24T20:45:47,653 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/WALs/20b981ef5c27,44901,1732481147262 2024-11-24T20:45:47,658 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T20:45:47,661 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=20b981ef5c27%2C44901%2C1732481147262, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/WALs/20b981ef5c27,44901,1732481147262, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/oldWALs, maxLogs=10 2024-11-24T20:45:47,678 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/WALs/20b981ef5c27,44901,1732481147262/20b981ef5c27%2C44901%2C1732481147262.1732481147662 2024-11-24T20:45:47,678 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T20:45:47,678 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:45:47,678 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:47,678 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,678 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T20:45:47,683 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:47,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:47,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T20:45:47,687 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:47,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:47,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T20:45:47,690 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:47,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:47,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T20:45:47,693 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:47,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:47,694 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,694 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,694 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,694 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,694 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,695 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T20:45:47,695 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:45:47,701 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:45:47,701 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67438574, jitterRate=0.004913061857223511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T20:45:47,702 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732481147679Initializing all the Stores at 1732481147679Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481147679Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481147679Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481147679Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481147679Cleaning up temporary data from old regions at 1732481147694 (+15 ms)Region opened successfully at 1732481147702 (+8 ms) 2024-11-24T20:45:47,702 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T20:45:47,706 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b27c749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=20b981ef5c27/172.17.0.2:0 2024-11-24T20:45:47,707 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T20:45:47,707 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T20:45:47,707 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T20:45:47,707 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T20:45:47,707 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T20:45:47,708 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-24T20:45:47,708 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T20:45:47,709 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T20:45:47,710 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T20:45:47,719 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T20:45:47,720 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T20:45:47,720 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T20:45:47,729 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T20:45:47,730 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T20:45:47,731 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T20:45:47,740 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T20:45:47,741 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T20:45:47,750 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T20:45:47,753 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T20:45:47,761 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T20:45:47,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:47,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:47,771 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:47,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,772 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,772 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=20b981ef5c27,44901,1732481147262, sessionid=0x1016e985d1d0000, setting cluster-up flag (Was=false) 2024-11-24T20:45:47,793 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,824 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T20:45:47,826 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=20b981ef5c27,44901,1732481147262 2024-11-24T20:45:47,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,845 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:47,877 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T20:45:47,878 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=20b981ef5c27,44901,1732481147262 2024-11-24T20:45:47,878 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T20:45:47,880 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T20:45:47,880 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T20:45:47,880 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T20:45:47,880 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 20b981ef5c27,44901,1732481147262 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T20:45:47,882 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:45:47,882 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:45:47,882 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:45:47,882 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:45:47,883 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/20b981ef5c27:0, corePoolSize=10, maxPoolSize=10 2024-11-24T20:45:47,883 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:47,883 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/20b981ef5c27:0, corePoolSize=2, maxPoolSize=2 2024-11-24T20:45:47,883 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:47,885 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732481177885 2024-11-24T20:45:47,885 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T20:45:47,885 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T20:45:47,885 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T20:45:47,885 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T20:45:47,885 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T20:45:47,885 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T20:45:47,885 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:47,886 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T20:45:47,886 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T20:45:47,886 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T20:45:47,886 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T20:45:47,886 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T20:45:47,886 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T20:45:47,886 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T20:45:47,886 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:47,886 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.large.0-1732481147886,5,FailOnTimeoutGroup] 2024-11-24T20:45:47,886 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.small.0-1732481147886,5,FailOnTimeoutGroup] 2024-11-24T20:45:47,887 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:47,887 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T20:45:47,887 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:47,887 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:47,886 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T20:45:47,902 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T20:45:47,902 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 2024-11-24T20:45:47,908 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:47,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T20:45:47,912 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T20:45:47,912 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:47,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:47,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T20:45:47,916 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T20:45:47,916 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:47,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:47,917 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(746): ClusterId : a868b0ac-f266-43e4-bc01-b920220e725d 2024-11-24T20:45:47,917 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(746): ClusterId : a868b0ac-f266-43e4-bc01-b920220e725d 2024-11-24T20:45:47,917 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T20:45:47,917 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T20:45:47,917 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T20:45:47,919 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T20:45:47,920 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:47,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:47,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T20:45:47,922 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T20:45:47,922 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:47,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:47,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T20:45:47,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740 2024-11-24T20:45:47,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740 2024-11-24T20:45:47,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T20:45:47,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T20:45:47,924 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T20:45:47,924 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T20:45:47,928 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:45:47,928 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61286244, jitterRate=-0.08676379919052124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T20:45:47,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732481147908Initializing all the Stores at 1732481147908Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481147908Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481147909 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481147909Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481147909Cleaning up temporary data from old regions at 1732481147923 (+14 ms)Region opened successfully at 1732481147929 (+6 ms) 2024-11-24T20:45:47,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T20:45:47,929 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T20:45:47,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T20:45:47,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T20:45:47,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T20:45:47,930 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T20:45:47,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732481147929Disabling compacts and flushes for region at 1732481147929Disabling writes for close at 1732481147929Writing region close event to WAL at 1732481147930 (+1 ms)Closed at 1732481147930 2024-11-24T20:45:47,931 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T20:45:47,931 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T20:45:47,931 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T20:45:47,931 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T20:45:47,931 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T20:45:47,931 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T20:45:47,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T20:45:47,933 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T20:45:47,934 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T20:45:47,941 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T20:45:47,941 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T20:45:47,941 DEBUG [RS:0;20b981ef5c27:43263 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2240bc41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=20b981ef5c27/172.17.0.2:0 2024-11-24T20:45:47,942 DEBUG [RS:1;20b981ef5c27:33461 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3123f962, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=20b981ef5c27/172.17.0.2:0 2024-11-24T20:45:47,953 DEBUG [RS:0;20b981ef5c27:43263 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;20b981ef5c27:43263 2024-11-24T20:45:47,953 DEBUG [RS:1;20b981ef5c27:33461 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;20b981ef5c27:33461 2024-11-24T20:45:47,953 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T20:45:47,953 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T20:45:47,953 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T20:45:47,953 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T20:45:47,953 DEBUG [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T20:45:47,953 DEBUG [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T20:45:47,954 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(2659): reportForDuty to master=20b981ef5c27,44901,1732481147262 with port=33461, startcode=1732481147481 2024-11-24T20:45:47,954 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(2659): reportForDuty to master=20b981ef5c27,44901,1732481147262 with port=43263, startcode=1732481147432 2024-11-24T20:45:47,954 DEBUG [RS:1;20b981ef5c27:33461 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T20:45:47,954 DEBUG [RS:0;20b981ef5c27:43263 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T20:45:47,957 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52205, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T20:45:47,957 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42475, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T20:45:47,957 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 20b981ef5c27,33461,1732481147481 2024-11-24T20:45:47,957 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44901 {}] master.ServerManager(517): Registering regionserver=20b981ef5c27,33461,1732481147481 2024-11-24T20:45:47,958 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 20b981ef5c27,43263,1732481147432 2024-11-24T20:45:47,959 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44901 {}] master.ServerManager(517): Registering regionserver=20b981ef5c27,43263,1732481147432 2024-11-24T20:45:47,959 DEBUG [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 2024-11-24T20:45:47,959 DEBUG [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T20:45:47,959 DEBUG [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T20:45:47,960 DEBUG [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 2024-11-24T20:45:47,960 DEBUG [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T20:45:47,960 DEBUG [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T20:45:47,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T20:45:48,012 DEBUG [RS:1;20b981ef5c27:33461 {}] zookeeper.ZKUtil(111): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/20b981ef5c27,33461,1732481147481 2024-11-24T20:45:48,012 WARN [RS:1;20b981ef5c27:33461 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T20:45:48,013 INFO [RS:1;20b981ef5c27:33461 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:45:48,013 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [20b981ef5c27,43263,1732481147432] 2024-11-24T20:45:48,013 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [20b981ef5c27,33461,1732481147481] 2024-11-24T20:45:48,013 DEBUG [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,33461,1732481147481 2024-11-24T20:45:48,013 DEBUG [RS:0;20b981ef5c27:43263 {}] zookeeper.ZKUtil(111): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/20b981ef5c27,43263,1732481147432 2024-11-24T20:45:48,014 WARN [RS:0;20b981ef5c27:43263 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T20:45:48,014 INFO [RS:0;20b981ef5c27:43263 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:45:48,014 DEBUG [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,43263,1732481147432 2024-11-24T20:45:48,023 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T20:45:48,025 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T20:45:48,025 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T20:45:48,026 INFO [RS:1;20b981ef5c27:33461 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T20:45:48,026 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,026 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T20:45:48,027 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T20:45:48,027 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,027 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,027 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,027 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,027 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,028 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,028 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T20:45:48,028 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/20b981ef5c27:0, corePoolSize=2, maxPoolSize=2 2024-11-24T20:45:48,028 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,028 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,028 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,028 INFO [RS:0;20b981ef5c27:43263 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T20:45:48,028 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,028 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,028 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,028 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,028 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:45:48,028 DEBUG [RS:1;20b981ef5c27:33461 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:45:48,029 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T20:45:48,030 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T20:45:48,030 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,030 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,030 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,030 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,030 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,030 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,030 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,030 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,030 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,33461,1732481147481-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T20:45:48,030 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,030 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,031 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,031 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/20b981ef5c27:0, corePoolSize=2, maxPoolSize=2 2024-11-24T20:45:48,031 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,031 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,031 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,031 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,031 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,031 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:48,031 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:45:48,032 DEBUG [RS:0;20b981ef5c27:43263 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:45:48,032 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,032 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,032 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,032 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,032 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,032 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,43263,1732481147432-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T20:45:48,046 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T20:45:48,046 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,33461,1732481147481-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,047 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,047 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.Replication(171): 20b981ef5c27,33461,1732481147481 started 2024-11-24T20:45:48,049 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T20:45:48,050 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,43263,1732481147432-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,050 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,050 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.Replication(171): 20b981ef5c27,43263,1732481147432 started 2024-11-24T20:45:48,060 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,061 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(1482): Serving as 20b981ef5c27,33461,1732481147481, RpcServer on 20b981ef5c27/172.17.0.2:33461, sessionid=0x1016e985d1d0002 2024-11-24T20:45:48,061 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T20:45:48,061 DEBUG [RS:1;20b981ef5c27:33461 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 20b981ef5c27,33461,1732481147481 2024-11-24T20:45:48,061 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,33461,1732481147481' 2024-11-24T20:45:48,061 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T20:45:48,062 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T20:45:48,063 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T20:45:48,063 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T20:45:48,063 DEBUG [RS:1;20b981ef5c27:33461 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 20b981ef5c27,33461,1732481147481 2024-11-24T20:45:48,063 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,33461,1732481147481' 2024-11-24T20:45:48,063 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T20:45:48,063 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T20:45:48,064 DEBUG [RS:1;20b981ef5c27:33461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T20:45:48,064 INFO [RS:1;20b981ef5c27:33461 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T20:45:48,064 INFO [RS:1;20b981ef5c27:33461 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T20:45:48,070 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,070 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(1482): Serving as 20b981ef5c27,43263,1732481147432, RpcServer on 20b981ef5c27/172.17.0.2:43263, sessionid=0x1016e985d1d0001 2024-11-24T20:45:48,070 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T20:45:48,070 DEBUG [RS:0;20b981ef5c27:43263 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 20b981ef5c27,43263,1732481147432 2024-11-24T20:45:48,070 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,43263,1732481147432' 2024-11-24T20:45:48,070 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T20:45:48,071 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T20:45:48,072 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T20:45:48,072 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T20:45:48,072 DEBUG [RS:0;20b981ef5c27:43263 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 20b981ef5c27,43263,1732481147432 2024-11-24T20:45:48,072 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,43263,1732481147432' 2024-11-24T20:45:48,072 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T20:45:48,072 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T20:45:48,073 DEBUG [RS:0;20b981ef5c27:43263 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T20:45:48,073 INFO [RS:0;20b981ef5c27:43263 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T20:45:48,073 INFO [RS:0;20b981ef5c27:43263 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T20:45:48,084 WARN [20b981ef5c27:44901 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T20:45:48,165 INFO [RS:1;20b981ef5c27:33461 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T20:45:48,166 INFO [RS:1;20b981ef5c27:33461 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=20b981ef5c27%2C33461%2C1732481147481, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,33461,1732481147481, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs, maxLogs=55 2024-11-24T20:45:48,173 INFO [RS:0;20b981ef5c27:43263 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T20:45:48,174 INFO [RS:0;20b981ef5c27:43263 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=20b981ef5c27%2C43263%2C1732481147432, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,43263,1732481147432, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs, maxLogs=55 2024-11-24T20:45:48,187 INFO [RS:1;20b981ef5c27:33461 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,33461,1732481147481/20b981ef5c27%2C33461%2C1732481147481.1732481148170 2024-11-24T20:45:48,187 DEBUG [RS:1;20b981ef5c27:33461 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T20:45:48,195 INFO [RS:0;20b981ef5c27:43263 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,43263,1732481147432/20b981ef5c27%2C43263%2C1732481147432.1732481148176 2024-11-24T20:45:48,195 DEBUG [RS:0;20b981ef5c27:43263 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T20:45:48,334 DEBUG [20b981ef5c27:44901 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T20:45:48,335 DEBUG [20b981ef5c27:44901 {}] balancer.BalancerClusterState(204): Hosts are {20b981ef5c27=0} racks are {/default-rack=0} 2024-11-24T20:45:48,336 DEBUG [20b981ef5c27:44901 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T20:45:48,336 DEBUG [20b981ef5c27:44901 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T20:45:48,336 DEBUG [20b981ef5c27:44901 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T20:45:48,336 DEBUG [20b981ef5c27:44901 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T20:45:48,337 INFO [20b981ef5c27:44901 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T20:45:48,337 INFO [20b981ef5c27:44901 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T20:45:48,337 DEBUG [20b981ef5c27:44901 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T20:45:48,337 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=20b981ef5c27,33461,1732481147481 2024-11-24T20:45:48,339 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 20b981ef5c27,33461,1732481147481, state=OPENING 2024-11-24T20:45:48,361 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T20:45:48,444 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:48,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:48,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:48,445 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T20:45:48,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=20b981ef5c27,33461,1732481147481}] 2024-11-24T20:45:48,446 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:48,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:48,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:48,603 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T20:45:48,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60385, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T20:45:48,607 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T20:45:48,607 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:45:48,608 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T20:45:48,610 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=20b981ef5c27%2C33461%2C1732481147481.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,33461,1732481147481, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs, maxLogs=55 2024-11-24T20:45:48,637 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,33461,1732481147481/20b981ef5c27%2C33461%2C1732481147481.meta.1732481148610.meta 2024-11-24T20:45:48,637 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T20:45:48,637 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:45:48,638 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T20:45:48,638 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T20:45:48,638 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T20:45:48,638 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T20:45:48,638 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:48,639 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T20:45:48,639 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T20:45:48,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T20:45:48,641 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T20:45:48,641 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:48,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:48,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T20:45:48,642 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T20:45:48,642 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:48,642 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:48,642 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T20:45:48,643 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T20:45:48,643 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:48,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:48,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T20:45:48,643 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T20:45:48,643 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:48,644 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:45:48,644 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T20:45:48,644 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740 2024-11-24T20:45:48,645 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740 2024-11-24T20:45:48,645 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T20:45:48,645 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T20:45:48,645 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T20:45:48,646 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T20:45:48,647 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69920846, jitterRate=0.04190179705619812}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T20:45:48,647 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T20:45:48,647 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732481148639Writing region info on filesystem at 1732481148639Initializing all the Stores at 1732481148639Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481148639Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481148640 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481148640Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481148640Cleaning up temporary data from old regions at 1732481148645 (+5 ms)Running coprocessor post-open hooks at 1732481148647 (+2 ms)Region opened successfully at 1732481148647 2024-11-24T20:45:48,649 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732481148602 2024-11-24T20:45:48,653 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T20:45:48,653 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T20:45:48,653 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,33461,1732481147481 2024-11-24T20:45:48,654 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 20b981ef5c27,33461,1732481147481, state=OPEN 2024-11-24T20:45:48,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T20:45:48,730 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T20:45:48,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T20:45:48,730 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=20b981ef5c27,33461,1732481147481 2024-11-24T20:45:48,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:48,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:48,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:45:48,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T20:45:48,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=20b981ef5c27,33461,1732481147481 in 284 msec 2024-11-24T20:45:48,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T20:45:48,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 802 msec 2024-11-24T20:45:48,736 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T20:45:48,736 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T20:45:48,739 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:45:48,739 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,33461,1732481147481, seqNum=-1] 2024-11-24T20:45:48,739 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:45:48,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48391, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:45:48,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 868 msec 2024-11-24T20:45:48,748 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732481148748, completionTime=-1 2024-11-24T20:45:48,748 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T20:45:48,748 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T20:45:48,751 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T20:45:48,751 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732481208751 2024-11-24T20:45:48,751 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732481268751 2024-11-24T20:45:48,751 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-24T20:45:48,752 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T20:45:48,752 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,44901,1732481147262-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,752 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,44901,1732481147262-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,752 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,44901,1732481147262-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,752 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-20b981ef5c27:44901, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,752 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,752 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,755 DEBUG [master/20b981ef5c27:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T20:45:48,757 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.196sec 2024-11-24T20:45:48,758 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T20:45:48,758 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T20:45:48,758 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T20:45:48,758 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T20:45:48,758 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T20:45:48,758 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,44901,1732481147262-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T20:45:48,758 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,44901,1732481147262-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T20:45:48,759 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T20:45:48,759 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T20:45:48,759 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,44901,1732481147262-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:48,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db2fb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:45:48,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,44901,-1 for getting cluster id 2024-11-24T20:45:48,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:45:48,820 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a868b0ac-f266-43e4-bc01-b920220e725d' 2024-11-24T20:45:48,820 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:45:48,820 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a868b0ac-f266-43e4-bc01-b920220e725d" 2024-11-24T20:45:48,820 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73503cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:45:48,820 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,44901,-1] 2024-11-24T20:45:48,821 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:45:48,821 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:48,823 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:45:48,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@765880bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:45:48,824 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:45:48,826 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,33461,1732481147481, seqNum=-1] 2024-11-24T20:45:48,826 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:45:48,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:45:48,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=20b981ef5c27,44901,1732481147262 2024-11-24T20:45:48,831 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T20:45:48,832 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 20b981ef5c27,44901,1732481147262 2024-11-24T20:45:48,832 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c93f974 2024-11-24T20:45:48,832 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T20:45:48,834 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T20:45:48,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44901 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732481148830', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:45:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44901 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732481148830 2024-11-24T20:45:48,838 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732481148830 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T20:45:48,838 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:48,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44901 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732481148830" procId is: 4 2024-11-24T20:45:48,839 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732481148830 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T20:45:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:45:48,848 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732481148830-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bb8c10a03a3b27874a8dd32338c4a530, NAME => 'testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732481148830', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 2024-11-24T20:45:48,856 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732481148830-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:48,856 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732481148830-pool-0 {}] regionserver.HRegion(1722): Closing bb8c10a03a3b27874a8dd32338c4a530, disabling compactions & flushes 2024-11-24T20:45:48,856 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732481148830-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:48,856 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732481148830-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:48,856 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732481148830-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. after waiting 0 ms 2024-11-24T20:45:48,856 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732481148830-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:48,856 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732481148830-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:48,857 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732481148830-pool-0 {}] regionserver.HRegion(1676): Region close journal for bb8c10a03a3b27874a8dd32338c4a530: Waiting for close lock at 1732481148856Disabling compacts and flushes for region at 1732481148856Disabling writes for close at 1732481148856Writing region close event to WAL at 1732481148856Closed at 1732481148856 2024-11-24T20:45:48,858 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732481148830 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T20:45:48,858 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732481148858"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732481148858"}]},"ts":"1732481148858"} 2024-11-24T20:45:48,860 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T20:45:48,861 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732481148830 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T20:45:48,862 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732481148830","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481148862"}]},"ts":"1732481148862"} 2024-11-24T20:45:48,864 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732481148830, state=ENABLING in hbase:meta 2024-11-24T20:45:48,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {20b981ef5c27=0} racks are {/default-rack=0} 2024-11-24T20:45:48,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T20:45:48,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T20:45:48,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T20:45:48,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T20:45:48,865 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T20:45:48,865 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T20:45:48,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T20:45:48,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732481148830, region=bb8c10a03a3b27874a8dd32338c4a530, ASSIGN}] 2024-11-24T20:45:48,867 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732481148830, region=bb8c10a03a3b27874a8dd32338c4a530, ASSIGN 2024-11-24T20:45:48,867 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732481148830, region=bb8c10a03a3b27874a8dd32338c4a530, ASSIGN; state=OFFLINE, location=20b981ef5c27,43263,1732481147432; forceNewPlan=false, retain=false 2024-11-24T20:45:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:45:49,018 INFO [20b981ef5c27:44901 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T20:45:49,019 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bb8c10a03a3b27874a8dd32338c4a530, regionState=OPENING, regionLocation=20b981ef5c27,43263,1732481147432 2024-11-24T20:45:49,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732481148830, region=bb8c10a03a3b27874a8dd32338c4a530, ASSIGN because future has completed 2024-11-24T20:45:49,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb8c10a03a3b27874a8dd32338c4a530, server=20b981ef5c27,43263,1732481147432}] 2024-11-24T20:45:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:45:49,178 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T20:45:49,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53085, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T20:45:49,182 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:49,183 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bb8c10a03a3b27874a8dd32338c4a530, NAME => 'testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:45:49,183 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732481148830 bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,183 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:49,184 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,184 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,185 INFO [StoreOpener-bb8c10a03a3b27874a8dd32338c4a530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,190 INFO [StoreOpener-bb8c10a03a3b27874a8dd32338c4a530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb8c10a03a3b27874a8dd32338c4a530 columnFamilyName family 2024-11-24T20:45:49,190 DEBUG [StoreOpener-bb8c10a03a3b27874a8dd32338c4a530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:49,190 INFO [StoreOpener-bb8c10a03a3b27874a8dd32338c4a530-1 {}] regionserver.HStore(327): Store=bb8c10a03a3b27874a8dd32338c4a530/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:49,190 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,192 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,192 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,193 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,193 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,194 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,201 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:45:49,202 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bb8c10a03a3b27874a8dd32338c4a530; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75094944, jitterRate=0.11900186538696289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T20:45:49,202 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:49,203 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bb8c10a03a3b27874a8dd32338c4a530: Running coprocessor pre-open hook at 1732481149184Writing region info on filesystem at 1732481149184Initializing all the Stores at 1732481149184Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481149185 (+1 ms)Cleaning up temporary data from old regions at 1732481149193 (+8 ms)Running coprocessor post-open hooks at 1732481149202 (+9 ms)Region opened successfully at 1732481149203 (+1 ms) 2024-11-24T20:45:49,203 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530., pid=6, masterSystemTime=1732481149177 2024-11-24T20:45:49,206 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:49,206 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:49,206 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bb8c10a03a3b27874a8dd32338c4a530, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,43263,1732481147432 2024-11-24T20:45:49,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb8c10a03a3b27874a8dd32338c4a530, server=20b981ef5c27,43263,1732481147432 because future has completed 2024-11-24T20:45:49,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T20:45:49,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bb8c10a03a3b27874a8dd32338c4a530, server=20b981ef5c27,43263,1732481147432 in 185 msec 2024-11-24T20:45:49,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T20:45:49,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732481148830, region=bb8c10a03a3b27874a8dd32338c4a530, ASSIGN in 347 msec 2024-11-24T20:45:49,214 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732481148830 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T20:45:49,214 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732481148830","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481149214"}]},"ts":"1732481149214"} 2024-11-24T20:45:49,216 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732481148830, state=ENABLED in hbase:meta 2024-11-24T20:45:49,217 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732481148830 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T20:45:49,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732481148830 in 381 msec 2024-11-24T20:45:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:45:49,469 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732481148830 completed 2024-11-24T20:45:49,470 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732481148830 get assigned. Timeout = 60000ms 2024-11-24T20:45:49,470 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T20:45:49,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732481148830 assigned to meta. Checking AM states. 2024-11-24T20:45:49,476 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T20:45:49,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732481148830 assigned. 2024-11-24T20:45:49,482 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T20:45:49,483 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732481148830', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530., hostname=20b981ef5c27,43263,1732481147432, seqNum=2] 2024-11-24T20:45:49,483 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:45:49,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:45:49,489 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bb8c10a03a3b27874a8dd32338c4a530 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T20:45:49,503 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4f7ad1fc0f6b496ab15344749d0a20c0 2024-11-24T20:45:49,523 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/.tmp/family/4f7ad1fc0f6b496ab15344749d0a20c0 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T20:45:49,523 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/.tmp/family/4f7ad1fc0f6b496ab15344749d0a20c0 2024-11-24T20:45:49,527 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/.tmp/family/4f7ad1fc0f6b496ab15344749d0a20c0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0 2024-11-24T20:45:49,529 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0, delay=936 ms 2024-11-24T20:45:49,530 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T20:45:49,531 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for bb8c10a03a3b27874a8dd32338c4a530 in 42ms, sequenceid=6, compaction requested=false 2024-11-24T20:45:49,531 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732481148830' 2024-11-24T20:45:49,532 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bb8c10a03a3b27874a8dd32338c4a530: 2024-11-24T20:45:50,466 DEBUG [hfile-prefetch-1732481146630 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4f7ad1fc0f6b496ab15344749d0a20c0_0. 2024-11-24T20:45:50,467 DEBUG [hfile-prefetch-1732481146630 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4f7ad1fc0f6b496ab15344749d0a20c0_117. 2024-11-24T20:45:50,467 DEBUG [hfile-prefetch-1732481146630 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0, with total blocks 2, and data blocks 1 2024-11-24T20:45:50,467 DEBUG [hfile-prefetch-1732481146630 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T20:45:50,467 DEBUG [hfile-prefetch-1732481146630 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T20:45:50,467 DEBUG [hfile-prefetch-1732481146630 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0. Will sleep 100ms and try the verification again. 2024-11-24T20:45:50,533 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;20b981ef5c27:43263,5,FailOnTimeoutGroup] 2024-11-24T20:45:50,533 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '20b981ef5c27,43263,1732481147432' ***** 2024-11-24T20:45:50,533 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-24T20:45:50,534 INFO [RS:0;20b981ef5c27:43263 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T20:45:50,535 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T20:45:50,535 INFO [regionserver/20b981ef5c27:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:45:50,567 DEBUG [hfile-prefetch-1732481146630 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0, with total blocks 2, and data blocks 1 2024-11-24T20:45:50,567 DEBUG [hfile-prefetch-1732481146630 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T20:45:50,567 DEBUG [hfile-prefetch-1732481146630 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T20:45:50,567 DEBUG [hfile-prefetch-1732481146630 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0. Will sleep 100ms and try the verification again. 2024-11-24T20:45:50,602 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T20:45:50,602 INFO [RS:0;20b981ef5c27:43263 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.persistence 2024-11-24T20:45:50,602 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T20:45:50,602 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T20:45:50,602 INFO [RS:0;20b981ef5c27:43263 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T20:45:50,603 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T20:45:50,607 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T20:45:50,607 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T20:45:50,607 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T20:45:50,607 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T20:45:50,610 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T20:45:50,610 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732481150607 2024-11-24T20:45:50,610 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T20:45:50,613 DEBUG [RS:0;20b981ef5c27:43263 {}] bucket.BucketCache(1487): Thread RS:0;20b981ef5c27:43263 started persisting bucket cache to file 2024-11-24T20:45:50,613 DEBUG [RS:0;20b981ef5c27:43263 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T20:45:50,613 DEBUG [RS:0;20b981ef5c27:43263 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T20:45:50,615 DEBUG [RS:0;20b981ef5c27:43263 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T20:45:50,615 DEBUG [RS:0;20b981ef5c27:43263 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732481150613 2024-11-24T20:45:50,615 DEBUG [RS:0;20b981ef5c27:43263 {}] bucket.BucketCache(1509): Thread RS:0;20b981ef5c27:43263 finished persisting bucket cache to file, renaming 2024-11-24T20:45:50,615 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T20:45:50,616 INFO [RS:0;20b981ef5c27:43263 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T20:45:50,616 INFO [RS:0;20b981ef5c27:43263 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T20:45:50,616 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(3091): Received CLOSE for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:50,616 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(959): stopping server 20b981ef5c27,43263,1732481147432 2024-11-24T20:45:50,616 INFO [RS:0;20b981ef5c27:43263 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T20:45:50,616 INFO [RS:0;20b981ef5c27:43263 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;20b981ef5c27:43263. 2024-11-24T20:45:50,616 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T20:45:50,617 DEBUG [RS:0;20b981ef5c27:43263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:45:50,617 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bb8c10a03a3b27874a8dd32338c4a530, disabling compactions & flushes 2024-11-24T20:45:50,617 DEBUG [RS:0;20b981ef5c27:43263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:50,617 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:50,617 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T20:45:50,617 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:50,617 DEBUG [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(1325): Online Regions={bb8c10a03a3b27874a8dd32338c4a530=testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530.} 2024-11-24T20:45:50,617 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. after waiting 0 ms 2024-11-24T20:45:50,617 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:50,617 DEBUG [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(1351): Waiting on bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:50,618 DEBUG [StoreFileCloser-bb8c10a03a3b27874a8dd32338c4a530-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0 2024-11-24T20:45:50,618 WARN [hfile-prefetch-1732481146630 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 10 more 2024-11-24T20:45:50,618 DEBUG [hfile-prefetch-1732481146630 {}] hfile.PrefetchExecutor(121): Prefetch completed for 4f7ad1fc0f6b496ab15344749d0a20c0 2024-11-24T20:45:50,623 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T20:45:50,624 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:50,624 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bb8c10a03a3b27874a8dd32338c4a530: Waiting for close lock at 1732481150616Running coprocessor pre-close hooks at 1732481150617 (+1 ms)Disabling compacts and flushes for region at 1732481150617Disabling writes for close at 1732481150617Writing region close event to WAL at 1732481150618 (+1 ms)Running coprocessor post-close hooks at 1732481150624 (+6 ms)Closed at 1732481150624 2024-11-24T20:45:50,624 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:50,817 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(976): stopping server 20b981ef5c27,43263,1732481147432; all regions closed. 2024-11-24T20:45:50,821 DEBUG [RS:0;20b981ef5c27:43263 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs 2024-11-24T20:45:50,821 INFO [RS:0;20b981ef5c27:43263 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 20b981ef5c27%2C43263%2C1732481147432:(num 1732481148176) 2024-11-24T20:45:50,821 DEBUG [RS:0;20b981ef5c27:43263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:50,821 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:45:50,822 INFO [RS:0;20b981ef5c27:43263 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T20:45:50,822 INFO [RS:0;20b981ef5c27:43263 {}] hbase.ChoreService(370): Chore service for: regionserver/20b981ef5c27:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T20:45:50,822 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T20:45:50,822 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T20:45:50,822 INFO [regionserver/20b981ef5c27:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T20:45:50,822 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T20:45:50,823 INFO [RS:0;20b981ef5c27:43263 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T20:45:50,823 INFO [RS:0;20b981ef5c27:43263 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43263 2024-11-24T20:45:50,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/20b981ef5c27,43263,1732481147432 2024-11-24T20:45:50,834 INFO [RS:0;20b981ef5c27:43263 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T20:45:50,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T20:45:50,845 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [20b981ef5c27,43263,1732481147432] 2024-11-24T20:45:50,856 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/20b981ef5c27,43263,1732481147432 already deleted, retry=false 2024-11-24T20:45:50,856 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 20b981ef5c27,43263,1732481147432 on 20b981ef5c27,44901,1732481147262 2024-11-24T20:45:50,863 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 20b981ef5c27,43263,1732481147432, splitWal=true, meta=false 2024-11-24T20:45:50,865 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 20b981ef5c27,43263,1732481147432 (carryingMeta=false) 20b981ef5c27,43263,1732481147432/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@14773029[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T20:45:50,866 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 20b981ef5c27,43263,1732481147432, splitWal=true, meta=false 2024-11-24T20:45:50,868 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 20b981ef5c27,43263,1732481147432 had 1 regions 2024-11-24T20:45:50,869 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 20b981ef5c27,43263,1732481147432, splitWal=true, meta=false, isMeta: false 2024-11-24T20:45:50,870 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,43263,1732481147432-splitting 2024-11-24T20:45:50,870 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,43263,1732481147432-splitting dir is empty, no logs to split. 2024-11-24T20:45:50,870 INFO [PEWorker-2 {}] master.SplitWALManager(105): 20b981ef5c27,43263,1732481147432 WAL count=0, meta=false 2024-11-24T20:45:50,873 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,43263,1732481147432-splitting dir is empty, no logs to split. 2024-11-24T20:45:50,873 INFO [PEWorker-2 {}] master.SplitWALManager(105): 20b981ef5c27,43263,1732481147432 WAL count=0, meta=false 2024-11-24T20:45:50,873 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 20b981ef5c27,43263,1732481147432 WAL splitting is done? wals=0, meta=false 2024-11-24T20:45:50,874 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 20b981ef5c27,43263,1732481147432 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,43263,1732481147432-splitting does not exist 2024-11-24T20:45:50,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732481148830, region=bb8c10a03a3b27874a8dd32338c4a530, ASSIGN}] 2024-11-24T20:45:50,877 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732481148830, region=bb8c10a03a3b27874a8dd32338c4a530, ASSIGN 2024-11-24T20:45:50,878 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732481148830, region=bb8c10a03a3b27874a8dd32338c4a530, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T20:45:50,945 INFO [RS:0;20b981ef5c27:43263 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T20:45:50,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:50,946 INFO [RS:0;20b981ef5c27:43263 {}] regionserver.HRegionServer(1031): Exiting; stopping=20b981ef5c27,43263,1732481147432; zookeeper connection closed. 2024-11-24T20:45:50,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43263-0x1016e985d1d0001, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:50,946 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7f9229b4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7f9229b4 2024-11-24T20:45:51,028 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bb8c10a03a3b27874a8dd32338c4a530, regionState=OPENING, regionLocation=20b981ef5c27,33461,1732481147481 2024-11-24T20:45:51,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732481148830, region=bb8c10a03a3b27874a8dd32338c4a530, ASSIGN because future has completed 2024-11-24T20:45:51,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb8c10a03a3b27874a8dd32338c4a530, server=20b981ef5c27,33461,1732481147481}] 2024-11-24T20:45:51,049 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/20b981ef5c27:0 server-side Connection retries=45 2024-11-24T20:45:51,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:51,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:51,050 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T20:45:51,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:45:51,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T20:45:51,050 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T20:45:51,050 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T20:45:51,051 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33739 2024-11-24T20:45:51,052 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33739 connecting to ZooKeeper ensemble=127.0.0.1:58231 2024-11-24T20:45:51,052 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:51,052 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:51,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337390x0, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T20:45:51,067 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33739-0x1016e985d1d0003 connected 2024-11-24T20:45:51,067 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33739-0x1016e985d1d0003, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-24T20:45:51,067 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-24T20:45:51,068 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T20:45:51,069 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.cache 2024-11-24T20:45:51,069 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T20:45:51,069 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T20:45:51,069 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T20:45:51,070 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T20:45:51,070 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T20:45:51,070 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T20:45:51,071 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-24T20:45:51,071 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33739-0x1016e985d1d0003, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T20:45:51,072 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-24T20:45:51,073 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33739-0x1016e985d1d0003, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T20:45:51,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33739 2024-11-24T20:45:51,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33739 2024-11-24T20:45:51,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33739 2024-11-24T20:45:51,078 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-24T20:45:51,078 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33739 2024-11-24T20:45:51,078 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33739 2024-11-24T20:45:51,079 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T20:45:51,079 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-24T20:45:51,079 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.persistence is complete. 2024-11-24T20:45:51,080 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(746): ClusterId : a868b0ac-f266-43e4-bc01-b920220e725d 2024-11-24T20:45:51,080 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T20:45:51,088 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T20:45:51,088 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T20:45:51,099 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T20:45:51,099 DEBUG [RS:2;20b981ef5c27:33739 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67512097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=20b981ef5c27/172.17.0.2:0 2024-11-24T20:45:51,111 DEBUG [RS:2;20b981ef5c27:33739 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;20b981ef5c27:33739 2024-11-24T20:45:51,111 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T20:45:51,111 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T20:45:51,111 DEBUG [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T20:45:51,112 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(2659): reportForDuty to master=20b981ef5c27,44901,1732481147262 with port=33739, startcode=1732481151049 2024-11-24T20:45:51,112 DEBUG [RS:2;20b981ef5c27:33739 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T20:45:51,114 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36937, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T20:45:51,115 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 20b981ef5c27,33739,1732481151049 2024-11-24T20:45:51,115 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44901 {}] master.ServerManager(517): Registering regionserver=20b981ef5c27,33739,1732481151049 2024-11-24T20:45:51,116 DEBUG [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99 2024-11-24T20:45:51,116 DEBUG [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T20:45:51,116 DEBUG [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T20:45:51,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T20:45:51,161 DEBUG [RS:2;20b981ef5c27:33739 {}] zookeeper.ZKUtil(111): regionserver:33739-0x1016e985d1d0003, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/20b981ef5c27,33739,1732481151049 2024-11-24T20:45:51,161 WARN [RS:2;20b981ef5c27:33739 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T20:45:51,162 INFO [RS:2;20b981ef5c27:33739 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:45:51,162 DEBUG [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,33739,1732481151049 2024-11-24T20:45:51,162 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [20b981ef5c27,33739,1732481151049] 2024-11-24T20:45:51,167 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T20:45:51,169 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T20:45:51,169 INFO [RS:2;20b981ef5c27:33739 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T20:45:51,169 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:51,169 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T20:45:51,170 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T20:45:51,170 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/20b981ef5c27:0, corePoolSize=2, maxPoolSize=2 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,170 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,171 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:45:51,171 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:45:51,171 DEBUG [RS:2;20b981ef5c27:33739 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:45:51,171 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:51,171 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:51,171 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:51,171 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:51,171 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:51,172 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,33739,1732481151049-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T20:45:51,186 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T20:45:51,186 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,33739,1732481151049-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:51,186 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:51,186 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.Replication(171): 20b981ef5c27,33739,1732481151049 started 2024-11-24T20:45:51,186 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:51,186 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => bb8c10a03a3b27874a8dd32338c4a530, NAME => 'testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:45:51,187 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732481148830 bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,187 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:45:51,187 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,187 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,188 INFO [StoreOpener-bb8c10a03a3b27874a8dd32338c4a530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,189 INFO [StoreOpener-bb8c10a03a3b27874a8dd32338c4a530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb8c10a03a3b27874a8dd32338c4a530 columnFamilyName family 2024-11-24T20:45:51,189 DEBUG [StoreOpener-bb8c10a03a3b27874a8dd32338c4a530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:45:51,190 DEBUG [StoreFileOpener-bb8c10a03a3b27874a8dd32338c4a530-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0, delay=993 ms 2024-11-24T20:45:51,191 DEBUG [StoreOpener-bb8c10a03a3b27874a8dd32338c4a530-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0 2024-11-24T20:45:51,192 INFO [StoreOpener-bb8c10a03a3b27874a8dd32338c4a530-1 {}] regionserver.HStore(327): Store=bb8c10a03a3b27874a8dd32338c4a530/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:45:51,192 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,192 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,193 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,193 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,193 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,194 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,194 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened bb8c10a03a3b27874a8dd32338c4a530; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64334800, jitterRate=-0.041336774826049805}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T20:45:51,195 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,195 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for bb8c10a03a3b27874a8dd32338c4a530: Running coprocessor pre-open hook at 1732481151187Writing region info on filesystem at 1732481151187Initializing all the Stores at 1732481151187Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481151187Cleaning up temporary data from old regions at 1732481151193 (+6 ms)Running coprocessor post-open hooks at 1732481151195 (+2 ms)Region opened successfully at 1732481151195 2024-11-24T20:45:51,196 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530., pid=9, masterSystemTime=1732481151183 2024-11-24T20:45:51,198 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:51,198 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:51,199 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bb8c10a03a3b27874a8dd32338c4a530, regionState=OPEN, openSeqNum=10, regionLocation=20b981ef5c27,33461,1732481147481 2024-11-24T20:45:51,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb8c10a03a3b27874a8dd32338c4a530, server=20b981ef5c27,33461,1732481147481 because future has completed 2024-11-24T20:45:51,204 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:45:51,204 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(1482): Serving as 20b981ef5c27,33739,1732481151049, RpcServer on 20b981ef5c27/172.17.0.2:33739, sessionid=0x1016e985d1d0003 2024-11-24T20:45:51,204 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T20:45:51,204 DEBUG [RS:2;20b981ef5c27:33739 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 20b981ef5c27,33739,1732481151049 2024-11-24T20:45:51,204 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,33739,1732481151049' 2024-11-24T20:45:51,204 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T20:45:51,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T20:45:51,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure bb8c10a03a3b27874a8dd32338c4a530, server=20b981ef5c27,33461,1732481147481 in 172 msec 2024-11-24T20:45:51,205 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T20:45:51,205 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T20:45:51,205 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T20:45:51,206 DEBUG [RS:2;20b981ef5c27:33739 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 20b981ef5c27,33739,1732481151049 2024-11-24T20:45:51,206 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,33739,1732481151049' 2024-11-24T20:45:51,206 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T20:45:51,206 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T20:45:51,207 DEBUG [RS:2;20b981ef5c27:33739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T20:45:51,207 INFO [RS:2;20b981ef5c27:33739 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T20:45:51,207 INFO [RS:2;20b981ef5c27:33739 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T20:45:51,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T20:45:51,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732481148830, region=bb8c10a03a3b27874a8dd32338c4a530, ASSIGN in 329 msec 2024-11-24T20:45:51,207 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 20b981ef5c27,43263,1732481147432 after splitting done 2024-11-24T20:45:51,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 20b981ef5c27,43263,1732481147432, splitWal=true, meta=false in 351 msec 2024-11-24T20:45:51,308 INFO [RS:2;20b981ef5c27:33739 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T20:45:51,309 INFO [RS:2;20b981ef5c27:33739 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=20b981ef5c27%2C33739%2C1732481151049, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,33739,1732481151049, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs, maxLogs=55 2024-11-24T20:45:51,331 INFO [RS:2;20b981ef5c27:33739 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/WALs/20b981ef5c27,33739,1732481151049/20b981ef5c27%2C33739%2C1732481151049.1732481151310 2024-11-24T20:45:51,331 DEBUG [RS:2;20b981ef5c27:33739 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T20:45:51,378 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T20:45:51,378 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T20:45:51,383 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732481148830 2024-11-24T20:45:51,383 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732481148830 Metrics about Tables on a single HBase RegionServer 2024-11-24T20:45:51,385 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T20:45:51,385 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T20:45:51,385 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T20:45:51,385 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T20:45:51,704 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T20:45:51,704 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:45:51,704 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:45:51,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:51,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:51,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:45:51,705 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T20:45:51,705 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1589713012, stopped=false 2024-11-24T20:45:51,705 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=20b981ef5c27,44901,1732481147262 2024-11-24T20:45:51,891 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:51,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33739-0x1016e985d1d0003, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:51,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T20:45:51,891 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:51,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:51,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33739-0x1016e985d1d0003, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:51,892 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T20:45:51,892 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:45:51,893 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:45:51,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:51,894 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:51,894 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '20b981ef5c27,33461,1732481147481' ***** 2024-11-24T20:45:51,894 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T20:45:51,894 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '20b981ef5c27,33739,1732481151049' ***** 2024-11-24T20:45:51,895 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T20:45:51,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:51,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33739-0x1016e985d1d0003, quorum=127.0.0.1:58231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:45:51,895 INFO [RS:2;20b981ef5c27:33739 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T20:45:51,896 INFO [RS:2;20b981ef5c27:33739 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.persistence 2024-11-24T20:45:51,896 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T20:45:51,896 INFO [RS:1;20b981ef5c27:33461 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T20:45:51,896 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T20:45:51,896 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T20:45:51,897 INFO [RS:2;20b981ef5c27:33739 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T20:45:51,897 INFO [RS:1;20b981ef5c27:33461 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cb9753c0-3f67-3f63-e42c-dd8cb17bf47b/bucket.persistence 2024-11-24T20:45:51,897 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T20:45:51,897 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T20:45:51,897 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T20:45:51,897 INFO [RS:1;20b981ef5c27:33461 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T20:45:51,897 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T20:45:51,897 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T20:45:51,907 DEBUG [RS:2;20b981ef5c27:33739 {}] bucket.BucketCache(1487): Thread RS:2;20b981ef5c27:33739 started persisting bucket cache to file 2024-11-24T20:45:51,907 DEBUG [RS:2;20b981ef5c27:33739 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T20:45:51,907 DEBUG [RS:2;20b981ef5c27:33739 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T20:45:51,907 DEBUG [RS:1;20b981ef5c27:33461 {}] bucket.BucketCache(1487): Thread RS:1;20b981ef5c27:33461 started persisting bucket cache to file 2024-11-24T20:45:51,907 DEBUG [RS:1;20b981ef5c27:33461 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T20:45:51,907 DEBUG [RS:1;20b981ef5c27:33461 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T20:45:51,910 DEBUG [RS:1;20b981ef5c27:33461 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T20:45:51,910 DEBUG [RS:1;20b981ef5c27:33461 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732481151907 2024-11-24T20:45:51,910 DEBUG [RS:2;20b981ef5c27:33739 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T20:45:51,910 DEBUG [RS:2;20b981ef5c27:33739 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732481151907 2024-11-24T20:45:51,910 DEBUG [RS:1;20b981ef5c27:33461 {}] bucket.BucketCache(1509): Thread RS:1;20b981ef5c27:33461 finished persisting bucket cache to file, renaming 2024-11-24T20:45:51,910 DEBUG [RS:2;20b981ef5c27:33739 {}] bucket.BucketCache(1509): Thread RS:2;20b981ef5c27:33739 finished persisting bucket cache to file, renaming 2024-11-24T20:45:51,910 WARN [RS:2;20b981ef5c27:33739 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-24T20:45:51,910 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T20:45:51,910 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T20:45:51,910 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T20:45:51,910 INFO [RS:1;20b981ef5c27:33461 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T20:45:51,910 INFO [RS:2;20b981ef5c27:33739 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T20:45:51,910 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T20:45:51,910 INFO [RS:1;20b981ef5c27:33461 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T20:45:51,910 INFO [RS:2;20b981ef5c27:33739 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T20:45:51,910 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(959): stopping server 20b981ef5c27,33739,1732481151049 2024-11-24T20:45:51,910 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(3091): Received CLOSE for bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,910 INFO [RS:2;20b981ef5c27:33739 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T20:45:51,911 INFO [RS:2;20b981ef5c27:33739 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;20b981ef5c27:33739. 2024-11-24T20:45:51,911 DEBUG [RS:2;20b981ef5c27:33739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:45:51,911 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(959): stopping server 20b981ef5c27,33461,1732481147481 2024-11-24T20:45:51,911 DEBUG [RS:2;20b981ef5c27:33739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:51,911 INFO [RS:1;20b981ef5c27:33461 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T20:45:51,911 INFO [RS:1;20b981ef5c27:33461 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;20b981ef5c27:33461. 2024-11-24T20:45:51,911 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(976): stopping server 20b981ef5c27,33739,1732481151049; all regions closed. 2024-11-24T20:45:51,911 DEBUG [RS:1;20b981ef5c27:33461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:45:51,911 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T20:45:51,911 DEBUG [RS:1;20b981ef5c27:33461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:51,911 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bb8c10a03a3b27874a8dd32338c4a530, disabling compactions & flushes 2024-11-24T20:45:51,911 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:51,911 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T20:45:51,911 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:51,911 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T20:45:51,911 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T20:45:51,911 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. after waiting 0 ms 2024-11-24T20:45:51,911 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T20:45:51,911 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:51,912 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T20:45:51,912 DEBUG [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(1325): Online Regions={bb8c10a03a3b27874a8dd32338c4a530=testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530., 1588230740=hbase:meta,,1.1588230740} 2024-11-24T20:45:51,912 DEBUG [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, bb8c10a03a3b27874a8dd32338c4a530 2024-11-24T20:45:51,912 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T20:45:51,912 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T20:45:51,912 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T20:45:51,912 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T20:45:51,912 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T20:45:51,912 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T20:45:51,913 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-24T20:45:51,913 DEBUG [RS:2;20b981ef5c27:33739 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs 2024-11-24T20:45:51,913 INFO [RS:2;20b981ef5c27:33739 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 20b981ef5c27%2C33739%2C1732481151049:(num 1732481151310) 2024-11-24T20:45:51,913 DEBUG [RS:2;20b981ef5c27:33739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:51,913 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:45:51,913 INFO [RS:2;20b981ef5c27:33739 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T20:45:51,914 INFO [RS:2;20b981ef5c27:33739 {}] hbase.ChoreService(370): Chore service for: regionserver/20b981ef5c27:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T20:45:51,914 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T20:45:51,914 DEBUG [StoreFileCloser-bb8c10a03a3b27874a8dd32338c4a530-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/family/4f7ad1fc0f6b496ab15344749d0a20c0 2024-11-24T20:45:51,914 INFO [regionserver/20b981ef5c27:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T20:45:51,914 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T20:45:51,914 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T20:45:51,914 INFO [RS:2;20b981ef5c27:33739 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T20:45:51,914 INFO [RS:2;20b981ef5c27:33739 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33739 2024-11-24T20:45:51,920 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/default/testBlockEvictionOnGracefulStop1732481148830/bb8c10a03a3b27874a8dd32338c4a530/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T20:45:51,920 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:51,920 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bb8c10a03a3b27874a8dd32338c4a530: Waiting for close lock at 1732481151911Running coprocessor pre-close hooks at 1732481151911Disabling compacts and flushes for region at 1732481151911Disabling writes for close at 1732481151911Writing region close event to WAL at 1732481151914 (+3 ms)Running coprocessor post-close hooks at 1732481151920 (+6 ms)Closed at 1732481151920 2024-11-24T20:45:51,921 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530. 2024-11-24T20:45:51,924 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8198bc4605be4430b21e784931c426fa 2024-11-24T20:45:51,934 INFO [regionserver/20b981ef5c27:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:45:51,939 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/info/8198bc4605be4430b21e784931c426fa is 209, key is testBlockEvictionOnGracefulStop1732481148830,,1732481148835.bb8c10a03a3b27874a8dd32338c4a530./info:regioninfo/1732481151199/Put/seqid=0 2024-11-24T20:45:51,939 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/info/8198bc4605be4430b21e784931c426fa 2024-11-24T20:45:51,947 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cfa6d69f687645f6ad076c618ed1aca5 2024-11-24T20:45:51,959 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/ns/cfa6d69f687645f6ad076c618ed1aca5 is 43, key is default/ns:d/1732481148742/Put/seqid=0 2024-11-24T20:45:51,960 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/ns/cfa6d69f687645f6ad076c618ed1aca5 2024-11-24T20:45:51,968 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9a0ee3e1f2a04d75aa03a14be07464ab 2024-11-24T20:45:51,972 INFO [regionserver/20b981ef5c27:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:45:51,980 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/table/9a0ee3e1f2a04d75aa03a14be07464ab is 80, key is testBlockEvictionOnGracefulStop1732481148830/table:state/1732481149214/Put/seqid=0 2024-11-24T20:45:51,981 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/table/9a0ee3e1f2a04d75aa03a14be07464ab 2024-11-24T20:45:51,983 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/info/8198bc4605be4430b21e784931c426fa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/info/8198bc4605be4430b21e784931c426fa 2024-11-24T20:45:51,984 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/info/8198bc4605be4430b21e784931c426fa, delay=976 ms 2024-11-24T20:45:51,985 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/info/8198bc4605be4430b21e784931c426fa, entries=14, sequenceid=13, filesize=7.5 K 2024-11-24T20:45:51,985 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/ns/cfa6d69f687645f6ad076c618ed1aca5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/ns/cfa6d69f687645f6ad076c618ed1aca5 2024-11-24T20:45:51,986 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/ns/cfa6d69f687645f6ad076c618ed1aca5, delay=913 ms 2024-11-24T20:45:51,988 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/ns/cfa6d69f687645f6ad076c618ed1aca5, entries=2, sequenceid=13, filesize=5.0 K 2024-11-24T20:45:51,988 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/.tmp/table/9a0ee3e1f2a04d75aa03a14be07464ab as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/table/9a0ee3e1f2a04d75aa03a14be07464ab 2024-11-24T20:45:51,990 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/table/9a0ee3e1f2a04d75aa03a14be07464ab, delay=941 ms 2024-11-24T20:45:51,991 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/table/9a0ee3e1f2a04d75aa03a14be07464ab, entries=2, sequenceid=13, filesize=5.3 K 2024-11-24T20:45:51,992 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=13, compaction requested=false 2024-11-24T20:45:51,993 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/info/8198bc4605be4430b21e784931c426fa 2024-11-24T20:45:51,993 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/ns/cfa6d69f687645f6ad076c618ed1aca5 2024-11-24T20:45:51,994 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/table/9a0ee3e1f2a04d75aa03a14be07464ab 2024-11-24T20:45:51,997 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-24T20:45:51,998 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T20:45:51,998 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T20:45:51,998 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732481151912Running coprocessor pre-close hooks at 1732481151912Disabling compacts and flushes for region at 1732481151912Disabling writes for close at 1732481151912Obtaining lock to block concurrent updates at 1732481151913 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732481151913Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732481151913Flushing stores of hbase:meta,,1.1588230740 at 1732481151914 (+1 ms)Flushing 1588230740/info: creating writer at 1732481151914Flushing 1588230740/info: appending metadata at 1732481151938 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732481151938Flushing 1588230740/ns: creating writer at 1732481151941 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732481151958 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732481151959 (+1 ms)Flushing 1588230740/table: creating writer at 1732481151962 (+3 ms)Flushing 1588230740/table: appending metadata at 1732481151979 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732481151979Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e355a59: reopening flushed file at 1732481151983 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cd33691: reopening flushed file at 1732481151985 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f4ad3a1: reopening flushed file at 1732481151988 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=13, compaction requested=false at 1732481151992 (+4 ms)Writing region close event to WAL at 1732481151994 (+2 ms)Running coprocessor post-close hooks at 1732481151998 (+4 ms)Closed at 1732481151998 2024-11-24T20:45:51,998 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T20:45:52,030 INFO [regionserver/20b981ef5c27:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T20:45:52,031 INFO [regionserver/20b981ef5c27:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T20:45:52,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T20:45:52,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33739-0x1016e985d1d0003, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/20b981ef5c27,33739,1732481151049 2024-11-24T20:45:52,060 INFO [RS:2;20b981ef5c27:33739 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T20:45:52,102 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [20b981ef5c27,33739,1732481151049] 2024-11-24T20:45:52,112 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(976): stopping server 20b981ef5c27,33461,1732481147481; all regions closed. 2024-11-24T20:45:52,116 DEBUG [RS:1;20b981ef5c27:33461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs 2024-11-24T20:45:52,116 INFO [RS:1;20b981ef5c27:33461 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 20b981ef5c27%2C33461%2C1732481147481.meta:.meta(num 1732481148610) 2024-11-24T20:45:52,119 DEBUG [RS:1;20b981ef5c27:33461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/oldWALs 2024-11-24T20:45:52,120 INFO [RS:1;20b981ef5c27:33461 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 20b981ef5c27%2C33461%2C1732481147481:(num 1732481148170) 2024-11-24T20:45:52,120 DEBUG [RS:1;20b981ef5c27:33461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:45:52,120 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:45:52,120 INFO [RS:1;20b981ef5c27:33461 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T20:45:52,121 INFO [RS:1;20b981ef5c27:33461 {}] hbase.ChoreService(370): Chore service for: regionserver/20b981ef5c27:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T20:45:52,121 INFO [RS:1;20b981ef5c27:33461 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T20:45:52,121 INFO [regionserver/20b981ef5c27:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T20:45:52,122 INFO [RS:1;20b981ef5c27:33461 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33461 2024-11-24T20:45:52,155 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/20b981ef5c27,33739,1732481151049 already deleted, retry=false 2024-11-24T20:45:52,155 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 20b981ef5c27,33739,1732481151049 expired; onlineServers=1 2024-11-24T20:45:52,182 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/20b981ef5c27,33461,1732481147481 2024-11-24T20:45:52,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T20:45:52,182 INFO [RS:1;20b981ef5c27:33461 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T20:45:52,192 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [20b981ef5c27,33461,1732481147481] 2024-11-24T20:45:52,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33739-0x1016e985d1d0003, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:52,203 INFO [RS:2;20b981ef5c27:33739 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T20:45:52,203 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/20b981ef5c27,33461,1732481147481 already deleted, retry=false 2024-11-24T20:45:52,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33739-0x1016e985d1d0003, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:52,203 INFO [RS:2;20b981ef5c27:33739 {}] regionserver.HRegionServer(1031): Exiting; stopping=20b981ef5c27,33739,1732481151049; zookeeper connection closed. 2024-11-24T20:45:52,203 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 20b981ef5c27,33461,1732481147481 expired; onlineServers=0 2024-11-24T20:45:52,203 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '20b981ef5c27,44901,1732481147262' ***** 2024-11-24T20:45:52,203 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T20:45:52,204 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@615971e3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@615971e3 2024-11-24T20:45:52,204 INFO [M:0;20b981ef5c27:44901 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T20:45:52,204 INFO [M:0;20b981ef5c27:44901 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T20:45:52,204 DEBUG [M:0;20b981ef5c27:44901 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T20:45:52,204 DEBUG [M:0;20b981ef5c27:44901 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T20:45:52,204 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T20:45:52,204 DEBUG [master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.small.0-1732481147886 {}] cleaner.HFileCleaner(306): Exit Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.small.0-1732481147886,5,FailOnTimeoutGroup] 2024-11-24T20:45:52,204 DEBUG [master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.large.0-1732481147886 {}] cleaner.HFileCleaner(306): Exit Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.large.0-1732481147886,5,FailOnTimeoutGroup] 2024-11-24T20:45:52,204 INFO [M:0;20b981ef5c27:44901 {}] hbase.ChoreService(370): Chore service for: master/20b981ef5c27:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T20:45:52,205 INFO [M:0;20b981ef5c27:44901 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T20:45:52,205 DEBUG [M:0;20b981ef5c27:44901 {}] master.HMaster(1795): Stopping service threads 2024-11-24T20:45:52,205 INFO [M:0;20b981ef5c27:44901 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T20:45:52,205 INFO [M:0;20b981ef5c27:44901 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T20:45:52,205 INFO [M:0;20b981ef5c27:44901 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T20:45:52,205 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T20:45:52,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T20:45:52,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:45:52,213 DEBUG [M:0;20b981ef5c27:44901 {}] zookeeper.ZKUtil(347): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T20:45:52,213 WARN [M:0;20b981ef5c27:44901 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T20:45:52,214 INFO [M:0;20b981ef5c27:44901 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/.lastflushedseqids 2024-11-24T20:45:52,220 INFO [M:0;20b981ef5c27:44901 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T20:45:52,220 INFO [M:0;20b981ef5c27:44901 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T20:45:52,220 DEBUG [M:0;20b981ef5c27:44901 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T20:45:52,220 INFO [M:0;20b981ef5c27:44901 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:52,220 DEBUG [M:0;20b981ef5c27:44901 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:52,220 DEBUG [M:0;20b981ef5c27:44901 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T20:45:52,220 DEBUG [M:0;20b981ef5c27:44901 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:52,220 INFO [M:0;20b981ef5c27:44901 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2024-11-24T20:45:52,229 DEBUG [M:0;20b981ef5c27:44901 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1ffc4d3e5d3346c0aaae123e8eba1435 2024-11-24T20:45:52,243 DEBUG [M:0;20b981ef5c27:44901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1ffc4d3e5d3346c0aaae123e8eba1435 is 82, key is hbase:meta,,1/info:regioninfo/1732481148653/Put/seqid=0 2024-11-24T20:45:52,243 INFO [M:0;20b981ef5c27:44901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1ffc4d3e5d3346c0aaae123e8eba1435 2024-11-24T20:45:52,250 DEBUG [M:0;20b981ef5c27:44901 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 41f659169ef24d4798369ac55496a964 2024-11-24T20:45:52,263 DEBUG [M:0;20b981ef5c27:44901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41f659169ef24d4798369ac55496a964 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732481149218/Put/seqid=0 2024-11-24T20:45:52,263 INFO [M:0;20b981ef5c27:44901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41f659169ef24d4798369ac55496a964 2024-11-24T20:45:52,270 DEBUG [M:0;20b981ef5c27:44901 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 73ecb5ba500540c889e67175e67ff250 2024-11-24T20:45:52,288 DEBUG [M:0;20b981ef5c27:44901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/73ecb5ba500540c889e67175e67ff250 is 69, key is 20b981ef5c27,33461,1732481147481/rs:state/1732481147957/Put/seqid=0 2024-11-24T20:45:52,289 INFO [M:0;20b981ef5c27:44901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/73ecb5ba500540c889e67175e67ff250 2024-11-24T20:45:52,291 INFO [M:0;20b981ef5c27:44901 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 73ecb5ba500540c889e67175e67ff250 2024-11-24T20:45:52,292 DEBUG [M:0;20b981ef5c27:44901 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1ffc4d3e5d3346c0aaae123e8eba1435 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1ffc4d3e5d3346c0aaae123e8eba1435 2024-11-24T20:45:52,293 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:52,293 INFO [RS:1;20b981ef5c27:33461 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T20:45:52,293 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33461-0x1016e985d1d0002, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:52,293 INFO [RS:1;20b981ef5c27:33461 {}] regionserver.HRegionServer(1031): Exiting; stopping=20b981ef5c27,33461,1732481147481; zookeeper connection closed. 2024-11-24T20:45:52,293 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@190d3e90 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@190d3e90 2024-11-24T20:45:52,293 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T20:45:52,294 INFO [M:0;20b981ef5c27:44901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1ffc4d3e5d3346c0aaae123e8eba1435, entries=8, sequenceid=83, filesize=5.5 K 2024-11-24T20:45:52,295 DEBUG [M:0;20b981ef5c27:44901 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41f659169ef24d4798369ac55496a964 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41f659169ef24d4798369ac55496a964 2024-11-24T20:45:52,297 INFO [M:0;20b981ef5c27:44901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41f659169ef24d4798369ac55496a964, entries=9, sequenceid=83, filesize=6.4 K 2024-11-24T20:45:52,297 DEBUG [M:0;20b981ef5c27:44901 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/73ecb5ba500540c889e67175e67ff250 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/73ecb5ba500540c889e67175e67ff250 2024-11-24T20:45:52,298 INFO [M:0;20b981ef5c27:44901 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 73ecb5ba500540c889e67175e67ff250 2024-11-24T20:45:52,299 INFO [M:0;20b981ef5c27:44901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83473219-35b9-7d85-c520-98913d9a7bc9/c4abcbcb-c3b0-2f9e-8f3b-4595ae0b6e99/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/73ecb5ba500540c889e67175e67ff250, entries=3, sequenceid=83, filesize=5.3 K 2024-11-24T20:45:52,299 INFO [M:0;20b981ef5c27:44901 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=83, compaction requested=false 2024-11-24T20:45:52,301 INFO [M:0;20b981ef5c27:44901 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:45:52,301 DEBUG [M:0;20b981ef5c27:44901 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732481152220Disabling compacts and flushes for region at 1732481152220Disabling writes for close at 1732481152220Obtaining lock to block concurrent updates at 1732481152220Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732481152220Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1732481152221 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732481152221Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732481152221Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732481152242 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732481152242Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732481152245 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732481152262 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732481152262Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732481152265 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732481152287 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732481152287Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b02bac2: reopening flushed file at 1732481152292 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@229749fc: reopening flushed file at 1732481152295 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@779f424a: reopening flushed file at 1732481152297 (+2 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=83, compaction requested=false at 1732481152299 (+2 ms)Writing region close event to WAL at 1732481152301 (+2 ms)Closed at 1732481152301 2024-11-24T20:45:52,302 INFO [M:0;20b981ef5c27:44901 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T20:45:52,302 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T20:45:52,303 INFO [M:0;20b981ef5c27:44901 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44901 2024-11-24T20:45:52,303 INFO [M:0;20b981ef5c27:44901 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T20:45:52,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:52,416 INFO [M:0;20b981ef5c27:44901 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T20:45:52,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44901-0x1016e985d1d0000, quorum=127.0.0.1:58231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:45:52,419 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T20:45:52,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T20:45:52,420 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:45:52,425 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 49) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=544 (was 556), ProcessCount=11 (was 11), AvailableMemoryMB=11057 (was 11200)