2024-11-22 18:44:57,850 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-22 18:44:57,863 main DEBUG Took 0.010439 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 18:44:57,863 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 18:44:57,863 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 18:44:57,871 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 18:44:57,873 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,882 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 18:44:57,898 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,901 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,902 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,904 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,905 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,906 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,907 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,908 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,908 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,909 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,909 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,910 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,911 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,912 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,912 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,913 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:44:57,914 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,914 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 18:44:57,916 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:44:57,918 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 18:44:57,920 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 18:44:57,921 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 18:44:57,922 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 18:44:57,923 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 18:44:57,935 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 18:44:57,938 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 18:44:57,941 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 18:44:57,941 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 18:44:57,941 main DEBUG createAppenders(={Console}) 2024-11-22 18:44:57,942 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-22 18:44:57,943 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-22 18:44:57,943 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-22 18:44:57,944 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 18:44:57,945 main DEBUG OutputStream closed 2024-11-22 18:44:57,945 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 18:44:57,945 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 18:44:57,946 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-22 18:44:58,014 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 18:44:58,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 18:44:58,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 18:44:58,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 18:44:58,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 18:44:58,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 18:44:58,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 18:44:58,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 18:44:58,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 18:44:58,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 18:44:58,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 18:44:58,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 18:44:58,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 18:44:58,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 18:44:58,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 18:44:58,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 18:44:58,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 18:44:58,023 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 18:44:58,025 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 18:44:58,025 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-22 18:44:58,025 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 18:44:58,026 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-22T18:44:58,040 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-22 18:44:58,043 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 18:44:58,043 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T18:44:58,274 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d 2024-11-22T18:44:58,296 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1016, ProcessCount=11, AvailableMemoryMB=8730 2024-11-22T18:44:58,397 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T18:44:58,487 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:44:58,501 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/cluster_d79da201-91a9-74a4-aca1-d8058d31e3a5, deleteOnExit=true 2024-11-22T18:44:58,587 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/cluster_d79da201-91a9-74a4-aca1-d8058d31e3a5/zookeeper_0, clientPort=56890, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/cluster_d79da201-91a9-74a4-aca1-d8058d31e3a5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/cluster_d79da201-91a9-74a4-aca1-d8058d31e3a5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T18:44:58,599 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56890 2024-11-22T18:44:58,600 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:44:58,684 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 with version=8 2024-11-22T18:44:58,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/hbase-staging 2024-11-22T18:44:58,780 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T18:44:59,074 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d79ba0c344fb:0 server-side Connection retries=45 2024-11-22T18:44:59,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:44:59,086 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T18:44:59,094 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T18:44:59,094 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:44:59,094 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T18:44:59,250 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T18:44:59,337 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T18:44:59,374 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T18:44:59,386 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T18:44:59,390 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T18:44:59,432 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15397 (auto-detected) 2024-11-22T18:44:59,434 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T18:44:59,457 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42489 2024-11-22T18:44:59,482 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42489 connecting to ZooKeeper ensemble=127.0.0.1:56890 2024-11-22T18:44:59,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:424890x0, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T18:44:59,519 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42489-0x10140fef8d70000 connected 2024-11-22T18:44:59,548 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:44:59,549 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:44:59,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:44:59,561 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9, hbase.cluster.distributed=false 2024-11-22T18:44:59,587 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T18:44:59,595 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42489 2024-11-22T18:44:59,595 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42489 2024-11-22T18:44:59,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42489 2024-11-22T18:44:59,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42489 2024-11-22T18:44:59,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42489 2024-11-22T18:44:59,684 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d79ba0c344fb:0 server-side Connection retries=45 2024-11-22T18:44:59,686 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:44:59,686 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T18:44:59,687 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T18:44:59,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:44:59,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T18:44:59,690 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T18:44:59,692 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T18:44:59,693 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42191 2024-11-22T18:44:59,695 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42191 connecting to ZooKeeper ensemble=127.0.0.1:56890 2024-11-22T18:44:59,696 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:44:59,697 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:44:59,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421910x0, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T18:44:59,701 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:421910x0, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:44:59,701 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42191-0x10140fef8d70001 connected 2024-11-22T18:44:59,707 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T18:44:59,719 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/bucket.cache 2024-11-22T18:44:59,725 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T18:44:59,726 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T18:44:59,728 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T18:44:59,730 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T18:44:59,730 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T18:44:59,736 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T18:44:59,742 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T18:44:59,743 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/bucket.persistence is complete. 2024-11-22T18:44:59,745 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T18:44:59,748 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T18:44:59,754 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T18:44:59,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42191 2024-11-22T18:44:59,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42191 2024-11-22T18:44:59,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42191 2024-11-22T18:44:59,762 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42191 2024-11-22T18:44:59,764 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42191 2024-11-22T18:44:59,783 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d79ba0c344fb:0 server-side Connection retries=45 2024-11-22T18:44:59,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:44:59,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T18:44:59,784 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T18:44:59,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:44:59,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T18:44:59,785 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T18:44:59,785 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T18:44:59,786 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43491 2024-11-22T18:44:59,788 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43491 connecting to ZooKeeper ensemble=127.0.0.1:56890 2024-11-22T18:44:59,788 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:44:59,789 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:44:59,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434910x0, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T18:44:59,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43491-0x10140fef8d70002 connected 2024-11-22T18:44:59,794 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:434910x0, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:44:59,794 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T18:44:59,797 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/bucket.cache 2024-11-22T18:44:59,797 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T18:44:59,797 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T18:44:59,798 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T18:44:59,804 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T18:44:59,805 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T18:44:59,805 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T18:44:59,805 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T18:44:59,805 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T18:44:59,806 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/bucket.persistence is complete. 2024-11-22T18:44:59,807 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T18:44:59,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T18:44:59,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43491 2024-11-22T18:44:59,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43491 2024-11-22T18:44:59,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43491 2024-11-22T18:44:59,816 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43491 2024-11-22T18:44:59,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43491 2024-11-22T18:44:59,841 DEBUG [M:0;d79ba0c344fb:42489 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d79ba0c344fb:42489 2024-11-22T18:44:59,842 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d79ba0c344fb,42489,1732301098869 2024-11-22T18:44:59,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:44:59,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:44:59,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:44:59,853 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d79ba0c344fb,42489,1732301098869 2024-11-22T18:44:59,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T18:44:59,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T18:44:59,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:44:59,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:44:59,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:44:59,887 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T18:44:59,889 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d79ba0c344fb,42489,1732301098869 from backup master directory 2024-11-22T18:44:59,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:44:59,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:44:59,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d79ba0c344fb,42489,1732301098869 2024-11-22T18:44:59,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:44:59,893 WARN [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T18:44:59,893 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d79ba0c344fb,42489,1732301098869 2024-11-22T18:44:59,896 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T18:44:59,899 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T18:44:59,907 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/hbase.id] with ID: 2e0b150f-f992-4d50-9445-caef2a25fc8f 2024-11-22T18:44:59,907 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/.tmp/hbase.id 2024-11-22T18:44:59,914 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/hbase.id] 2024-11-22T18:44:59,959 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:44:59,965 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T18:44:59,993 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-22T18:44:59,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:44:59,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:44:59,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:00,025 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:45:00,028 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T18:45:00,075 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:45:00,081 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:45:00,140 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store 2024-11-22T18:45:00,163 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T18:45:00,166 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:00,168 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T18:45:00,168 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:00,168 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:00,170 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T18:45:00,170 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:00,171 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:00,171 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732301100168Disabling compacts and flushes for region at 1732301100168Disabling writes for close at 1732301100170 (+2 ms)Writing region close event to WAL at 1732301100171 (+1 ms)Closed at 1732301100171 2024-11-22T18:45:00,172 WARN [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/.initializing 2024-11-22T18:45:00,172 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/WALs/d79ba0c344fb,42489,1732301098869 2024-11-22T18:45:00,187 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T18:45:00,199 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d79ba0c344fb%2C42489%2C1732301098869, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/WALs/d79ba0c344fb,42489,1732301098869, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/oldWALs, maxLogs=10 2024-11-22T18:45:00,238 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/WALs/d79ba0c344fb,42489,1732301098869/d79ba0c344fb%2C42489%2C1732301098869.1732301100204 2024-11-22T18:45:00,239 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T18:45:00,240 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:45:00,241 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:00,245 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,247 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,291 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,328 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T18:45:00,334 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:00,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:00,337 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,343 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T18:45:00,343 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:00,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:00,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T18:45:00,349 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:00,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:00,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,356 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T18:45:00,356 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:00,357 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:00,357 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,362 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,363 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,367 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,368 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,372 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T18:45:00,375 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:00,383 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:45:00,384 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70941112, jitterRate=0.05710494518280029}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T18:45:00,394 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732301100266Initializing all the Stores at 1732301100266Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301100267 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301100268 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301100268Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301100269 (+1 ms)Cleaning up temporary data from old regions at 1732301100368 (+99 ms)Region opened successfully at 1732301100394 (+26 ms) 2024-11-22T18:45:00,395 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T18:45:00,438 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7128c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d79ba0c344fb/172.17.0.2:0 2024-11-22T18:45:00,483 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T18:45:00,495 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T18:45:00,496 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T18:45:00,499 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T18:45:00,499 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T18:45:00,505 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T18:45:00,506 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T18:45:00,531 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T18:45:00,541 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T18:45:00,543 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T18:45:00,546 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T18:45:00,548 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T18:45:00,550 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T18:45:00,553 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T18:45:00,557 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T18:45:00,558 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T18:45:00,560 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T18:45:00,562 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T18:45:00,583 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T18:45:00,585 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T18:45:00,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:00,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:00,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:00,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:00,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:00,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:00,602 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d79ba0c344fb,42489,1732301098869, sessionid=0x10140fef8d70000, setting cluster-up flag (Was=false) 2024-11-22T18:45:00,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:00,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:00,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:00,623 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T18:45:00,625 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d79ba0c344fb,42489,1732301098869 2024-11-22T18:45:00,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:00,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:00,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:00,638 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T18:45:00,640 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d79ba0c344fb,42489,1732301098869 2024-11-22T18:45:00,645 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T18:45:00,717 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T18:45:00,722 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(746): ClusterId : 2e0b150f-f992-4d50-9445-caef2a25fc8f 2024-11-22T18:45:00,722 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(746): ClusterId : 2e0b150f-f992-4d50-9445-caef2a25fc8f 2024-11-22T18:45:00,725 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T18:45:00,725 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T18:45:00,730 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T18:45:00,730 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T18:45:00,730 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T18:45:00,730 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T18:45:00,731 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T18:45:00,734 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T18:45:00,734 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T18:45:00,735 DEBUG [RS:1;d79ba0c344fb:43491 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787b852a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d79ba0c344fb/172.17.0.2:0 2024-11-22T18:45:00,735 DEBUG [RS:0;d79ba0c344fb:42191 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1d45ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d79ba0c344fb/172.17.0.2:0 2024-11-22T18:45:00,742 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T18:45:00,748 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d79ba0c344fb,42489,1732301098869 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T18:45:00,757 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:45:00,757 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:45:00,757 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:45:00,758 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:45:00,758 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d79ba0c344fb:0, corePoolSize=10, maxPoolSize=10 2024-11-22T18:45:00,758 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,758 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=2, maxPoolSize=2 2024-11-22T18:45:00,758 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,759 DEBUG [RS:0;d79ba0c344fb:42191 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d79ba0c344fb:42191 2024-11-22T18:45:00,759 DEBUG [RS:1;d79ba0c344fb:43491 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d79ba0c344fb:43491 2024-11-22T18:45:00,761 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732301130761 2024-11-22T18:45:00,762 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T18:45:00,762 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T18:45:00,762 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T18:45:00,762 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T18:45:00,763 DEBUG [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T18:45:00,763 DEBUG [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T18:45:00,763 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T18:45:00,765 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T18:45:00,765 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T18:45:00,765 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T18:45:00,768 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(2659): reportForDuty to master=d79ba0c344fb,42489,1732301098869 with port=43491, startcode=1732301099783 2024-11-22T18:45:00,768 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(2659): reportForDuty to master=d79ba0c344fb,42489,1732301098869 with port=42191, startcode=1732301099646 2024-11-22T18:45:00,769 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T18:45:00,769 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T18:45:00,770 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T18:45:00,770 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T18:45:00,770 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:00,770 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T18:45:00,772 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,776 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T18:45:00,778 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T18:45:00,778 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T18:45:00,785 DEBUG [RS:0;d79ba0c344fb:42191 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T18:45:00,785 DEBUG [RS:1;d79ba0c344fb:43491 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T18:45:00,785 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T18:45:00,786 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T18:45:00,788 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.large.0-1732301100787,5,FailOnTimeoutGroup] 2024-11-22T18:45:00,789 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.small.0-1732301100789,5,FailOnTimeoutGroup] 2024-11-22T18:45:00,789 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,789 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T18:45:00,791 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,791 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,794 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T18:45:00,794 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 2024-11-22T18:45:00,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:00,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T18:45:00,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T18:45:00,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:00,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:00,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T18:45:00,821 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T18:45:00,821 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:00,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:00,822 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T18:45:00,828 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T18:45:00,829 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:00,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:00,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T18:45:00,836 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T18:45:00,836 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:00,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:00,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T18:45:00,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740 2024-11-22T18:45:00,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740 2024-11-22T18:45:00,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T18:45:00,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T18:45:00,839 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T18:45:00,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T18:45:00,847 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:45:00,848 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69203412, jitterRate=0.03121119737625122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T18:45:00,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732301100803Initializing all the Stores at 1732301100803Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301100804 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301100804Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301100804Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301100804Cleaning up temporary data from old regions at 1732301100839 (+35 ms)Region opened successfully at 1732301100851 (+12 ms) 2024-11-22T18:45:00,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T18:45:00,851 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T18:45:00,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T18:45:00,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T18:45:00,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T18:45:00,853 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T18:45:00,854 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732301100851Disabling compacts and flushes for region at 1732301100851Disabling writes for close at 1732301100852 (+1 ms)Writing region close event to WAL at 1732301100853 (+1 ms)Closed at 1732301100853 2024-11-22T18:45:00,857 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T18:45:00,857 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T18:45:00,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T18:45:00,868 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32865, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T18:45:00,868 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58267, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T18:45:00,874 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T18:45:00,876 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42489 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:00,877 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T18:45:00,879 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42489 {}] master.ServerManager(517): Registering regionserver=d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:00,888 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42489 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:00,888 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42489 {}] master.ServerManager(517): Registering regionserver=d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:00,892 DEBUG [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 2024-11-22T18:45:00,892 DEBUG [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 2024-11-22T18:45:00,892 DEBUG [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T18:45:00,892 DEBUG [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T18:45:00,892 DEBUG [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T18:45:00,892 DEBUG [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T18:45:00,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T18:45:00,903 DEBUG [RS:1;d79ba0c344fb:43491 {}] zookeeper.ZKUtil(111): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:00,903 DEBUG [RS:0;d79ba0c344fb:42191 {}] zookeeper.ZKUtil(111): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:00,903 WARN [RS:1;d79ba0c344fb:43491 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T18:45:00,903 WARN [RS:0;d79ba0c344fb:42191 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T18:45:00,903 INFO [RS:1;d79ba0c344fb:43491 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:45:00,903 INFO [RS:0;d79ba0c344fb:42191 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:45:00,904 DEBUG [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:00,904 DEBUG [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:00,905 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d79ba0c344fb,43491,1732301099783] 2024-11-22T18:45:00,905 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d79ba0c344fb,42191,1732301099646] 2024-11-22T18:45:00,932 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T18:45:00,932 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T18:45:00,947 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T18:45:00,947 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T18:45:00,952 INFO [RS:0;d79ba0c344fb:42191 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T18:45:00,952 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,953 INFO [RS:1;d79ba0c344fb:43491 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T18:45:00,953 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,953 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T18:45:00,953 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T18:45:00,959 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T18:45:00,959 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T18:45:00,961 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,961 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,961 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,961 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,961 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,961 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,962 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,962 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,962 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,962 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,962 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,962 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,962 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d79ba0c344fb:0, corePoolSize=2, maxPoolSize=2 2024-11-22T18:45:00,962 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d79ba0c344fb:0, corePoolSize=2, maxPoolSize=2 2024-11-22T18:45:00,962 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,962 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,963 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,963 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,963 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,963 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,963 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,963 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,963 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,963 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,964 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,964 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:00,964 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:45:00,964 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:45:00,964 DEBUG [RS:1;d79ba0c344fb:43491 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:45:00,964 DEBUG [RS:0;d79ba0c344fb:42191 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:45:00,967 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,967 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,967 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,968 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,968 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,968 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,968 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,968 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,42191,1732301099646-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T18:45:00,968 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,968 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,969 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,969 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,43491,1732301099783-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T18:45:00,988 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T18:45:00,990 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T18:45:00,990 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,42191,1732301099646-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,990 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,43491,1732301099783-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,991 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,991 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:00,991 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.Replication(171): d79ba0c344fb,42191,1732301099646 started 2024-11-22T18:45:00,991 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.Replication(171): d79ba0c344fb,43491,1732301099783 started 2024-11-22T18:45:01,010 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:01,010 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:01,010 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(1482): Serving as d79ba0c344fb,42191,1732301099646, RpcServer on d79ba0c344fb/172.17.0.2:42191, sessionid=0x10140fef8d70001 2024-11-22T18:45:01,010 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(1482): Serving as d79ba0c344fb,43491,1732301099783, RpcServer on d79ba0c344fb/172.17.0.2:43491, sessionid=0x10140fef8d70002 2024-11-22T18:45:01,011 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T18:45:01,011 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T18:45:01,011 DEBUG [RS:0;d79ba0c344fb:42191 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:01,011 DEBUG [RS:1;d79ba0c344fb:43491 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:01,011 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,42191,1732301099646' 2024-11-22T18:45:01,012 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,43491,1732301099783' 2024-11-22T18:45:01,012 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T18:45:01,012 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T18:45:01,013 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T18:45:01,013 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T18:45:01,013 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T18:45:01,013 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T18:45:01,013 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T18:45:01,013 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T18:45:01,014 DEBUG [RS:0;d79ba0c344fb:42191 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:01,014 DEBUG [RS:1;d79ba0c344fb:43491 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:01,014 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,42191,1732301099646' 2024-11-22T18:45:01,014 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,43491,1732301099783' 2024-11-22T18:45:01,014 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T18:45:01,014 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T18:45:01,014 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T18:45:01,015 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T18:45:01,015 DEBUG [RS:0;d79ba0c344fb:42191 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T18:45:01,015 DEBUG [RS:1;d79ba0c344fb:43491 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T18:45:01,015 INFO [RS:0;d79ba0c344fb:42191 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T18:45:01,015 INFO [RS:1;d79ba0c344fb:43491 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T18:45:01,015 INFO [RS:0;d79ba0c344fb:42191 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T18:45:01,015 INFO [RS:1;d79ba0c344fb:43491 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T18:45:01,028 WARN [d79ba0c344fb:42489 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T18:45:01,121 INFO [RS:0;d79ba0c344fb:42191 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T18:45:01,121 INFO [RS:1;d79ba0c344fb:43491 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T18:45:01,122 INFO [RS:1;d79ba0c344fb:43491 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d79ba0c344fb%2C43491%2C1732301099783, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,43491,1732301099783, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs, maxLogs=55 2024-11-22T18:45:01,122 INFO [RS:0;d79ba0c344fb:42191 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d79ba0c344fb%2C42191%2C1732301099646, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,42191,1732301099646, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs, maxLogs=55 2024-11-22T18:45:01,146 INFO [RS:0;d79ba0c344fb:42191 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,42191,1732301099646/d79ba0c344fb%2C42191%2C1732301099646.1732301101127 2024-11-22T18:45:01,147 DEBUG [RS:0;d79ba0c344fb:42191 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T18:45:01,150 INFO [RS:1;d79ba0c344fb:43491 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,43491,1732301099783/d79ba0c344fb%2C43491%2C1732301099783.1732301101127 2024-11-22T18:45:01,151 DEBUG [RS:1;d79ba0c344fb:43491 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T18:45:01,280 DEBUG [d79ba0c344fb:42489 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-22T18:45:01,291 DEBUG [d79ba0c344fb:42489 {}] balancer.BalancerClusterState(204): Hosts are {d79ba0c344fb=0} racks are {/default-rack=0} 2024-11-22T18:45:01,295 DEBUG [d79ba0c344fb:42489 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T18:45:01,295 DEBUG [d79ba0c344fb:42489 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T18:45:01,295 DEBUG [d79ba0c344fb:42489 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T18:45:01,295 DEBUG [d79ba0c344fb:42489 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T18:45:01,295 INFO [d79ba0c344fb:42489 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T18:45:01,295 INFO [d79ba0c344fb:42489 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T18:45:01,295 DEBUG [d79ba0c344fb:42489 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T18:45:01,303 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:01,308 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d79ba0c344fb,42191,1732301099646, state=OPENING 2024-11-22T18:45:01,313 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T18:45:01,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:01,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:01,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:01,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:01,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:01,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:01,318 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T18:45:01,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d79ba0c344fb,42191,1732301099646}] 2024-11-22T18:45:01,491 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T18:45:01,495 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55131, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T18:45:01,502 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T18:45:01,502 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:45:01,503 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T18:45:01,504 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d79ba0c344fb%2C42191%2C1732301099646.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,42191,1732301099646, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs, maxLogs=55 2024-11-22T18:45:01,524 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,42191,1732301099646/d79ba0c344fb%2C42191%2C1732301099646.meta.1732301101506.meta 2024-11-22T18:45:01,525 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T18:45:01,525 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:45:01,527 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T18:45:01,529 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T18:45:01,534 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T18:45:01,538 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T18:45:01,538 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:01,538 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T18:45:01,539 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T18:45:01,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T18:45:01,541 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T18:45:01,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:01,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:01,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T18:45:01,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T18:45:01,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:01,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:01,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T18:45:01,543 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T18:45:01,543 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:01,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:01,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T18:45:01,545 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T18:45:01,545 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:01,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:01,545 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T18:45:01,546 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740 2024-11-22T18:45:01,547 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740 2024-11-22T18:45:01,547 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T18:45:01,547 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T18:45:01,547 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T18:45:01,548 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T18:45:01,549 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63684854, jitterRate=-0.051021724939346313}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T18:45:01,549 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T18:45:01,551 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732301101539Writing region info on filesystem at 1732301101539Initializing all the Stores at 1732301101539Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301101540 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301101540Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301101540Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301101540Cleaning up temporary data from old regions at 1732301101547 (+7 ms)Running coprocessor post-open hooks at 1732301101549 (+2 ms)Region opened successfully at 1732301101550 (+1 ms) 2024-11-22T18:45:01,557 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732301101483 2024-11-22T18:45:01,568 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T18:45:01,568 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T18:45:01,569 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:01,570 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d79ba0c344fb,42191,1732301099646, state=OPEN 2024-11-22T18:45:01,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T18:45:01,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T18:45:01,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T18:45:01,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:01,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:01,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:01,573 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:01,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T18:45:01,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d79ba0c344fb,42191,1732301099646 in 254 msec 2024-11-22T18:45:01,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T18:45:01,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 715 msec 2024-11-22T18:45:01,584 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T18:45:01,584 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T18:45:01,605 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:45:01,606 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,42191,1732301099646, seqNum=-1] 2024-11-22T18:45:01,626 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:45:01,630 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33753, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:45:01,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 975 msec 2024-11-22T18:45:01,648 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732301101648, completionTime=-1 2024-11-22T18:45:01,651 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-22T18:45:01,651 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T18:45:01,678 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T18:45:01,678 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732301161678 2024-11-22T18:45:01,679 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732301221679 2024-11-22T18:45:01,679 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-22T18:45:01,681 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-22T18:45:01,687 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,42489,1732301098869-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:01,687 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,42489,1732301098869-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:01,687 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,42489,1732301098869-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:01,689 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d79ba0c344fb:42489, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:01,689 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:01,690 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:01,695 DEBUG [master/d79ba0c344fb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T18:45:01,716 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.822sec 2024-11-22T18:45:01,717 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T18:45:01,719 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T18:45:01,719 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T18:45:01,720 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T18:45:01,720 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T18:45:01,721 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,42489,1732301098869-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T18:45:01,721 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,42489,1732301098869-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T18:45:01,723 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T18:45:01,723 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T18:45:01,724 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,42489,1732301098869-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:01,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73434faf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:45:01,733 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T18:45:01,733 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T18:45:01,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,42489,-1 for getting cluster id 2024-11-22T18:45:01,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:45:01,748 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e0b150f-f992-4d50-9445-caef2a25fc8f' 2024-11-22T18:45:01,751 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:45:01,751 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e0b150f-f992-4d50-9445-caef2a25fc8f" 2024-11-22T18:45:01,754 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776b0beb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:45:01,754 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,42489,-1] 2024-11-22T18:45:01,757 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:45:01,758 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:01,760 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:45:01,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@969eb34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:45:01,763 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:45:01,770 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,42191,1732301099646, seqNum=-1] 2024-11-22T18:45:01,770 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:45:01,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:45:01,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d79ba0c344fb,42489,1732301098869 2024-11-22T18:45:01,796 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T18:45:01,815 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d79ba0c344fb,42489,1732301098869 2024-11-22T18:45:01,818 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5bd5631f 2024-11-22T18:45:01,819 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T18:45:01,822 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T18:45:01,830 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732301101792', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:45:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732301101792 2024-11-22T18:45:01,840 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732301101792 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T18:45:01,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732301101792" procId is: 4 2024-11-22T18:45:01,843 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:01,844 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732301101792 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T18:45:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:45:01,858 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732301101792-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9221b2b774783280c366ded9047a143c, NAME => 'testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732301101792', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 2024-11-22T18:45:01,864 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732301101792-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:01,864 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732301101792-pool-0 {}] regionserver.HRegion(1722): Closing 9221b2b774783280c366ded9047a143c, disabling compactions & flushes 2024-11-22T18:45:01,864 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732301101792-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:01,864 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732301101792-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:01,865 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732301101792-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. after waiting 0 ms 2024-11-22T18:45:01,865 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732301101792-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:01,865 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732301101792-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:01,865 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732301101792-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9221b2b774783280c366ded9047a143c: Waiting for close lock at 1732301101864Disabling compacts and flushes for region at 1732301101864Disabling writes for close at 1732301101865 (+1 ms)Writing region close event to WAL at 1732301101865Closed at 1732301101865 2024-11-22T18:45:01,866 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732301101792 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T18:45:01,871 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732301101866"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732301101866"}]},"ts":"1732301101866"} 2024-11-22T18:45:01,876 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T18:45:01,877 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732301101792 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T18:45:01,881 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732301101792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301101877"}]},"ts":"1732301101877"} 2024-11-22T18:45:01,887 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732301101792, state=ENABLING in hbase:meta 2024-11-22T18:45:01,888 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(204): Hosts are {d79ba0c344fb=0} racks are {/default-rack=0} 2024-11-22T18:45:01,889 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T18:45:01,889 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T18:45:01,889 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T18:45:01,889 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T18:45:01,889 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T18:45:01,889 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T18:45:01,889 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T18:45:01,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, ASSIGN}] 2024-11-22T18:45:01,892 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, ASSIGN 2024-11-22T18:45:01,893 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, ASSIGN; state=OFFLINE, location=d79ba0c344fb,42191,1732301099646; forceNewPlan=false, retain=false 2024-11-22T18:45:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:45:02,045 INFO [d79ba0c344fb:42489 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T18:45:02,046 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9221b2b774783280c366ded9047a143c, regionState=OPENING, regionLocation=d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:02,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, ASSIGN because future has completed 2024-11-22T18:45:02,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9221b2b774783280c366ded9047a143c, server=d79ba0c344fb,42191,1732301099646}] 2024-11-22T18:45:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:45:02,208 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:02,209 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9221b2b774783280c366ded9047a143c, NAME => 'testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:45:02,209 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732301101792 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,209 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:02,209 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,209 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,210 INFO [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,215 INFO [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9221b2b774783280c366ded9047a143c columnFamilyName family 2024-11-22T18:45:02,215 DEBUG [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:02,216 INFO [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] regionserver.HStore(327): Store=9221b2b774783280c366ded9047a143c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:02,216 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,217 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,217 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,217 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,217 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,218 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,224 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:45:02,225 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9221b2b774783280c366ded9047a143c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66169109, jitterRate=-0.014003440737724304}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T18:45:02,225 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:02,226 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9221b2b774783280c366ded9047a143c: Running coprocessor pre-open hook at 1732301102210Writing region info on filesystem at 1732301102210Initializing all the Stores at 1732301102210Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301102210Cleaning up temporary data from old regions at 1732301102217 (+7 ms)Running coprocessor post-open hooks at 1732301102225 (+8 ms)Region opened successfully at 1732301102226 (+1 ms) 2024-11-22T18:45:02,227 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c., pid=6, masterSystemTime=1732301102204 2024-11-22T18:45:02,230 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:02,230 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:02,231 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9221b2b774783280c366ded9047a143c, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:02,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9221b2b774783280c366ded9047a143c, server=d79ba0c344fb,42191,1732301099646 because future has completed 2024-11-22T18:45:02,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T18:45:02,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9221b2b774783280c366ded9047a143c, server=d79ba0c344fb,42191,1732301099646 in 184 msec 2024-11-22T18:45:02,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T18:45:02,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, ASSIGN in 347 msec 2024-11-22T18:45:02,241 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732301101792 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T18:45:02,241 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732301101792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301102241"}]},"ts":"1732301102241"} 2024-11-22T18:45:02,243 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732301101792, state=ENABLED in hbase:meta 2024-11-22T18:45:02,244 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732301101792 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T18:45:02,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732301101792 in 411 msec 2024-11-22T18:45:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:45:02,478 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732301101792 completed 2024-11-22T18:45:02,478 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732301101792 get assigned. Timeout = 60000ms 2024-11-22T18:45:02,481 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T18:45:02,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732301101792 assigned to meta. Checking AM states. 2024-11-22T18:45:02,487 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T18:45:02,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732301101792 assigned. 2024-11-22T18:45:02,496 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732301101792', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c., hostname=d79ba0c344fb,42191,1732301099646, seqNum=2] 2024-11-22T18:45:02,508 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9221b2b774783280c366ded9047a143c 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T18:45:02,538 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f42be50578a54a849d54ea10aaf19c00 2024-11-22T18:45:02,577 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/.tmp/family/f42be50578a54a849d54ea10aaf19c00 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T18:45:02,583 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/.tmp/family/f42be50578a54a849d54ea10aaf19c00 2024-11-22T18:45:02,623 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/.tmp/family/f42be50578a54a849d54ea10aaf19c00 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00 2024-11-22T18:45:02,625 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00, delay=964 ms 2024-11-22T18:45:02,628 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T18:45:02,637 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 9221b2b774783280c366ded9047a143c in 126ms, sequenceid=6, compaction requested=false 2024-11-22T18:45:02,638 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732301101792' 2024-11-22T18:45:02,641 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9221b2b774783280c366ded9047a143c: 2024-11-22T18:45:03,592 DEBUG [hfile-prefetch-1732301102626 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f42be50578a54a849d54ea10aaf19c00_0. 2024-11-22T18:45:03,596 DEBUG [hfile-prefetch-1732301102626 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f42be50578a54a849d54ea10aaf19c00_117. 2024-11-22T18:45:03,597 DEBUG [hfile-prefetch-1732301102626 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00, with total blocks 2, and data blocks 1 2024-11-22T18:45:03,597 DEBUG [hfile-prefetch-1732301102626 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T18:45:03,597 DEBUG [hfile-prefetch-1732301102626 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T18:45:03,598 DEBUG [hfile-prefetch-1732301102626 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00. Will sleep 100ms and try the verification again. 2024-11-22T18:45:03,611 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T18:45:03,611 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T18:45:03,612 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T18:45:03,612 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T18:45:03,628 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T18:45:03,628 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732301103611 2024-11-22T18:45:03,628 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T18:45:03,651 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=9221b2b774783280c366ded9047a143c, source=d79ba0c344fb,42191,1732301099646, destination=d79ba0c344fb,43491,1732301099783, warming up region on d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T18:45:03,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=9221b2b774783280c366ded9047a143c, source=d79ba0c344fb,42191,1732301099646, destination=d79ba0c344fb,43491,1732301099783, running balancer 2024-11-22T18:45:03,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36763, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T18:45:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, REOPEN/MOVE 2024-11-22T18:45:03,656 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, REOPEN/MOVE 2024-11-22T18:45:03,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43491 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43491 {}] regionserver.HRegion(7855): Warmup {ENCODED => 9221b2b774783280c366ded9047a143c, NAME => 'testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:45:03,657 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9221b2b774783280c366ded9047a143c, regionState=CLOSING, regionLocation=d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43491 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:03,658 INFO [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:03,659 INFO [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9221b2b774783280c366ded9047a143c columnFamilyName family 2024-11-22T18:45:03,659 DEBUG [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:03,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, REOPEN/MOVE because future has completed 2024-11-22T18:45:03,661 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-22T18:45:03,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9221b2b774783280c366ded9047a143c, server=d79ba0c344fb,42191,1732301099646}] 2024-11-22T18:45:03,662 DEBUG [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00 2024-11-22T18:45:03,662 INFO [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] regionserver.HStore(327): Store=9221b2b774783280c366ded9047a143c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43491 {}] regionserver.HRegion(1722): Closing 9221b2b774783280c366ded9047a143c, disabling compactions & flushes 2024-11-22T18:45:03,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43491 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43491 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43491 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. after waiting 0 ms 2024-11-22T18:45:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43491 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:03,664 DEBUG [StoreFileCloser-9221b2b774783280c366ded9047a143c-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00 2024-11-22T18:45:03,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43491 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43491 {}] regionserver.HRegion(1676): Region close journal for 9221b2b774783280c366ded9047a143c: Waiting for close lock at 1732301103663Disabling compacts and flushes for region at 1732301103663Disabling writes for close at 1732301103663Writing region close event to WAL at 1732301103666 (+3 ms)Closed at 1732301103666 2024-11-22T18:45:03,667 WARN [hfile-prefetch-1732301102626 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-22T18:45:03,668 DEBUG [hfile-prefetch-1732301102626 {}] hfile.PrefetchExecutor(121): Prefetch completed for f42be50578a54a849d54ea10aaf19c00 2024-11-22T18:45:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-22T18:45:03,819 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:03,820 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-22T18:45:03,820 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 9221b2b774783280c366ded9047a143c, disabling compactions & flushes 2024-11-22T18:45:03,820 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:03,821 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:03,821 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. after waiting 0 ms 2024-11-22T18:45:03,821 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:03,822 DEBUG [StoreFileCloser-9221b2b774783280c366ded9047a143c-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file f42be50578a54a849d54ea10aaf19c00, starting offset: 0, end offset: 9223372036854775807 2024-11-22T18:45:03,824 DEBUG [StoreFileCloser-9221b2b774783280c366ded9047a143c-family-1 {}] bucket.BucketCache(856): removed key f42be50578a54a849d54ea10aaf19c00_0 from back map with offset lock 2094080 in the evict process 2024-11-22T18:45:03,829 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T18:45:03,829 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T18:45:03,829 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T18:45:03,829 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T18:45:03,829 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T18:45:03,830 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:03,831 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 9221b2b774783280c366ded9047a143c: Waiting for close lock at 1732301103820Running coprocessor pre-close hooks at 1732301103820Disabling compacts and flushes for region at 1732301103820Disabling writes for close at 1732301103821 (+1 ms)Writing region close event to WAL at 1732301103824 (+3 ms)Running coprocessor post-close hooks at 1732301103829 (+5 ms)Closed at 1732301103830 (+1 ms) 2024-11-22T18:45:03,831 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 9221b2b774783280c366ded9047a143c move to d79ba0c344fb,43491,1732301099783 record at close sequenceid=6 2024-11-22T18:45:03,832 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T18:45:03,832 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T18:45:03,832 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T18:45:03,832 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T18:45:03,832 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T18:45:03,832 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732301103829 2024-11-22T18:45:03,832 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T18:45:03,835 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T18:45:03,835 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9221b2b774783280c366ded9047a143c, regionState=CLOSED 2024-11-22T18:45:03,835 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732301103832 2024-11-22T18:45:03,835 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T18:45:03,836 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:03,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9221b2b774783280c366ded9047a143c, server=d79ba0c344fb,42191,1732301099646 because future has completed 2024-11-22T18:45:03,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T18:45:03,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 9221b2b774783280c366ded9047a143c, server=d79ba0c344fb,42191,1732301099646 in 178 msec 2024-11-22T18:45:03,842 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, REOPEN/MOVE; state=CLOSED, location=d79ba0c344fb,43491,1732301099783; forceNewPlan=false, retain=false 2024-11-22T18:45:03,992 INFO [d79ba0c344fb:42489 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T18:45:03,993 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9221b2b774783280c366ded9047a143c, regionState=OPENING, regionLocation=d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:03,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, REOPEN/MOVE because future has completed 2024-11-22T18:45:03,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9221b2b774783280c366ded9047a143c, server=d79ba0c344fb,43491,1732301099783}] 2024-11-22T18:45:04,152 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:04,153 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 9221b2b774783280c366ded9047a143c, NAME => 'testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:45:04,153 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732301101792 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,153 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:04,153 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,153 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,154 INFO [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,155 INFO [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9221b2b774783280c366ded9047a143c columnFamilyName family 2024-11-22T18:45:04,156 DEBUG [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:04,157 DEBUG [StoreFileOpener-9221b2b774783280c366ded9047a143c-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00, delay=964 ms 2024-11-22T18:45:04,159 DEBUG [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00 2024-11-22T18:45:04,159 INFO [StoreOpener-9221b2b774783280c366ded9047a143c-1 {}] regionserver.HStore(327): Store=9221b2b774783280c366ded9047a143c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:04,159 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,160 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,160 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,160 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,161 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,165 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,165 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 9221b2b774783280c366ded9047a143c; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69278155, jitterRate=0.032324954867362976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T18:45:04,166 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,166 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 9221b2b774783280c366ded9047a143c: Running coprocessor pre-open hook at 1732301104154Writing region info on filesystem at 1732301104154Initializing all the Stores at 1732301104154Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301104154Cleaning up temporary data from old regions at 1732301104161 (+7 ms)Running coprocessor post-open hooks at 1732301104166 (+5 ms)Region opened successfully at 1732301104166 2024-11-22T18:45:04,167 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c., pid=9, masterSystemTime=1732301104149 2024-11-22T18:45:04,170 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:04,170 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:04,170 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9221b2b774783280c366ded9047a143c, regionState=OPEN, openSeqNum=10, regionLocation=d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:04,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9221b2b774783280c366ded9047a143c, server=d79ba0c344fb,43491,1732301099783 because future has completed 2024-11-22T18:45:04,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-22T18:45:04,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9221b2b774783280c366ded9047a143c, server=d79ba0c344fb,43491,1732301099783 in 177 msec 2024-11-22T18:45:04,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732301101792, region=9221b2b774783280c366ded9047a143c, REOPEN/MOVE in 522 msec 2024-11-22T18:45:04,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T18:45:04,209 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:45:04,209 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:45:04,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:04,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:04,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:45:04,214 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T18:45:04,214 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=632300009, stopped=false 2024-11-22T18:45:04,214 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d79ba0c344fb,42489,1732301098869 2024-11-22T18:45:04,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:04,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:04,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:04,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:04,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:04,217 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T18:45:04,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:04,217 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:45:04,218 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:45:04,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:04,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:45:04,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:45:04,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:45:04,218 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd79ba0c344fb,42191,1732301099646' ***** 2024-11-22T18:45:04,218 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T18:45:04,218 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd79ba0c344fb,43491,1732301099783' ***** 2024-11-22T18:45:04,218 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T18:45:04,219 INFO [RS:0;d79ba0c344fb:42191 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T18:45:04,219 INFO [RS:1;d79ba0c344fb:43491 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T18:45:04,219 INFO [RS:0;d79ba0c344fb:42191 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/bucket.persistence 2024-11-22T18:45:04,219 INFO [RS:1;d79ba0c344fb:43491 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/bucket.persistence 2024-11-22T18:45:04,219 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T18:45:04,219 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T18:45:04,219 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T18:45:04,219 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T18:45:04,219 INFO [RS:0;d79ba0c344fb:42191 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T18:45:04,220 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T18:45:04,220 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T18:45:04,220 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T18:45:04,220 INFO [RS:1;d79ba0c344fb:43491 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T18:45:04,220 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T18:45:04,230 DEBUG [RS:0;d79ba0c344fb:42191 {}] bucket.BucketCache(1487): Thread RS:0;d79ba0c344fb:42191 started persisting bucket cache to file 2024-11-22T18:45:04,230 DEBUG [RS:0;d79ba0c344fb:42191 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T18:45:04,230 DEBUG [RS:1;d79ba0c344fb:43491 {}] bucket.BucketCache(1487): Thread RS:1;d79ba0c344fb:43491 started persisting bucket cache to file 2024-11-22T18:45:04,230 DEBUG [RS:0;d79ba0c344fb:42191 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T18:45:04,230 DEBUG [RS:1;d79ba0c344fb:43491 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T18:45:04,230 DEBUG [RS:1;d79ba0c344fb:43491 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T18:45:04,234 DEBUG [RS:1;d79ba0c344fb:43491 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T18:45:04,234 DEBUG [RS:1;d79ba0c344fb:43491 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732301104230 2024-11-22T18:45:04,234 DEBUG [RS:0;d79ba0c344fb:42191 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T18:45:04,234 DEBUG [RS:0;d79ba0c344fb:42191 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732301104230 2024-11-22T18:45:04,234 DEBUG [RS:1;d79ba0c344fb:43491 {}] bucket.BucketCache(1509): Thread RS:1;d79ba0c344fb:43491 finished persisting bucket cache to file, renaming 2024-11-22T18:45:04,234 DEBUG [RS:0;d79ba0c344fb:42191 {}] bucket.BucketCache(1509): Thread RS:0;d79ba0c344fb:42191 finished persisting bucket cache to file, renaming 2024-11-22T18:45:04,234 WARN [RS:0;d79ba0c344fb:42191 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-22T18:45:04,234 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T18:45:04,234 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T18:45:04,235 INFO [RS:1;d79ba0c344fb:43491 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T18:45:04,235 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T18:45:04,235 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T18:45:04,235 INFO [RS:0;d79ba0c344fb:42191 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T18:45:04,235 INFO [RS:1;d79ba0c344fb:43491 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T18:45:04,235 INFO [RS:0;d79ba0c344fb:42191 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T18:45:04,235 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(3091): Received CLOSE for 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,235 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(959): stopping server d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:04,235 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T18:45:04,235 INFO [RS:0;d79ba0c344fb:42191 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d79ba0c344fb:42191. 2024-11-22T18:45:04,235 DEBUG [RS:0;d79ba0c344fb:42191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:45:04,235 DEBUG [RS:0;d79ba0c344fb:42191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:04,236 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(959): stopping server d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:04,236 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T18:45:04,236 INFO [RS:1;d79ba0c344fb:43491 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d79ba0c344fb:43491. 2024-11-22T18:45:04,236 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T18:45:04,236 DEBUG [RS:1;d79ba0c344fb:43491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:45:04,236 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T18:45:04,236 DEBUG [RS:1;d79ba0c344fb:43491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:04,236 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T18:45:04,236 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T18:45:04,236 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T18:45:04,236 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T18:45:04,236 DEBUG [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(1325): Online Regions={9221b2b774783280c366ded9047a143c=testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c.} 2024-11-22T18:45:04,236 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T18:45:04,237 DEBUG [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T18:45:04,237 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9221b2b774783280c366ded9047a143c, disabling compactions & flushes 2024-11-22T18:45:04,237 DEBUG [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(1351): Waiting on 9221b2b774783280c366ded9047a143c 2024-11-22T18:45:04,237 DEBUG [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T18:45:04,237 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T18:45:04,237 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:04,237 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:04,237 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T18:45:04,237 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. after waiting 0 ms 2024-11-22T18:45:04,237 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T18:45:04,237 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:04,237 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T18:45:04,237 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T18:45:04,237 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T18:45:04,237 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-22T18:45:04,238 DEBUG [StoreFileCloser-9221b2b774783280c366ded9047a143c-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/family/f42be50578a54a849d54ea10aaf19c00 2024-11-22T18:45:04,243 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnRegionMove1732301101792/9221b2b774783280c366ded9047a143c/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-22T18:45:04,243 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:04,243 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9221b2b774783280c366ded9047a143c: Waiting for close lock at 1732301104236Running coprocessor pre-close hooks at 1732301104237 (+1 ms)Disabling compacts and flushes for region at 1732301104237Disabling writes for close at 1732301104237Writing region close event to WAL at 1732301104238 (+1 ms)Running coprocessor post-close hooks at 1732301104243 (+5 ms)Closed at 1732301104243 2024-11-22T18:45:04,243 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c. 2024-11-22T18:45:04,252 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 21ddd5504a7a40df9d9fda285bb13678 2024-11-22T18:45:04,270 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/info/21ddd5504a7a40df9d9fda285bb13678 is 205, key is testBlockEvictionOnRegionMove1732301101792,,1732301101823.9221b2b774783280c366ded9047a143c./info:regioninfo/1732301104170/Put/seqid=0 2024-11-22T18:45:04,271 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/info/21ddd5504a7a40df9d9fda285bb13678 2024-11-22T18:45:04,272 INFO [regionserver/d79ba0c344fb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:45:04,273 INFO [regionserver/d79ba0c344fb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:45:04,281 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b8545f70771e4c9aa33b6301265cfb6e 2024-11-22T18:45:04,296 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/ns/b8545f70771e4c9aa33b6301265cfb6e is 43, key is default/ns:d/1732301101634/Put/seqid=0 2024-11-22T18:45:04,297 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/ns/b8545f70771e4c9aa33b6301265cfb6e 2024-11-22T18:45:04,306 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6b2fc5bc4ddc4cbd957aa1ef1e6d3f16 2024-11-22T18:45:04,319 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/table/6b2fc5bc4ddc4cbd957aa1ef1e6d3f16 is 78, key is testBlockEvictionOnRegionMove1732301101792/table:state/1732301102241/Put/seqid=0 2024-11-22T18:45:04,320 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/table/6b2fc5bc4ddc4cbd957aa1ef1e6d3f16 2024-11-22T18:45:04,322 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/info/21ddd5504a7a40df9d9fda285bb13678 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/info/21ddd5504a7a40df9d9fda285bb13678 2024-11-22T18:45:04,324 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/info/21ddd5504a7a40df9d9fda285bb13678, delay=909 ms 2024-11-22T18:45:04,325 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/info/21ddd5504a7a40df9d9fda285bb13678, entries=15, sequenceid=15, filesize=7.6 K 2024-11-22T18:45:04,325 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/ns/b8545f70771e4c9aa33b6301265cfb6e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/ns/b8545f70771e4c9aa33b6301265cfb6e 2024-11-22T18:45:04,327 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/ns/b8545f70771e4c9aa33b6301265cfb6e, delay=941 ms 2024-11-22T18:45:04,328 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/ns/b8545f70771e4c9aa33b6301265cfb6e, entries=2, sequenceid=15, filesize=5.0 K 2024-11-22T18:45:04,328 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/table/6b2fc5bc4ddc4cbd957aa1ef1e6d3f16 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/table/6b2fc5bc4ddc4cbd957aa1ef1e6d3f16 2024-11-22T18:45:04,330 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/table/6b2fc5bc4ddc4cbd957aa1ef1e6d3f16, delay=903 ms 2024-11-22T18:45:04,331 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/table/6b2fc5bc4ddc4cbd957aa1ef1e6d3f16, entries=2, sequenceid=15, filesize=5.3 K 2024-11-22T18:45:04,332 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=15, compaction requested=false 2024-11-22T18:45:04,332 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T18:45:04,333 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/info/21ddd5504a7a40df9d9fda285bb13678 2024-11-22T18:45:04,334 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/ns/b8545f70771e4c9aa33b6301265cfb6e 2024-11-22T18:45:04,335 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/table/6b2fc5bc4ddc4cbd957aa1ef1e6d3f16 2024-11-22T18:45:04,339 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-22T18:45:04,340 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T18:45:04,340 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T18:45:04,340 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732301104237Running coprocessor pre-close hooks at 1732301104237Disabling compacts and flushes for region at 1732301104237Disabling writes for close at 1732301104237Obtaining lock to block concurrent updates at 1732301104237Preparing flush snapshotting stores in 1588230740 at 1732301104237Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732301104238 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732301104238Flushing 1588230740/info: creating writer at 1732301104239 (+1 ms)Flushing 1588230740/info: appending metadata at 1732301104269 (+30 ms)Flushing 1588230740/info: closing flushed file at 1732301104269Flushing 1588230740/ns: creating writer at 1732301104274 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732301104295 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732301104295Flushing 1588230740/table: creating writer at 1732301104299 (+4 ms)Flushing 1588230740/table: appending metadata at 1732301104318 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732301104319 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@426217d8: reopening flushed file at 1732301104322 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7965f8ca: reopening flushed file at 1732301104325 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ba3c078: reopening flushed file at 1732301104328 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=15, compaction requested=false at 1732301104332 (+4 ms)Writing region close event to WAL at 1732301104335 (+3 ms)Running coprocessor post-close hooks at 1732301104340 (+5 ms)Closed at 1732301104340 2024-11-22T18:45:04,341 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T18:45:04,437 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(976): stopping server d79ba0c344fb,43491,1732301099783; all regions closed. 2024-11-22T18:45:04,437 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(976): stopping server d79ba0c344fb,42191,1732301099646; all regions closed. 2024-11-22T18:45:04,443 DEBUG [RS:0;d79ba0c344fb:42191 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs 2024-11-22T18:45:04,443 DEBUG [RS:1;d79ba0c344fb:43491 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs 2024-11-22T18:45:04,443 INFO [RS:1;d79ba0c344fb:43491 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d79ba0c344fb%2C43491%2C1732301099783:(num 1732301101127) 2024-11-22T18:45:04,443 INFO [RS:0;d79ba0c344fb:42191 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d79ba0c344fb%2C42191%2C1732301099646.meta:.meta(num 1732301101506) 2024-11-22T18:45:04,443 DEBUG [RS:1;d79ba0c344fb:43491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:04,443 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:45:04,443 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T18:45:04,444 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.ChoreService(370): Chore service for: regionserver/d79ba0c344fb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T18:45:04,444 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T18:45:04,444 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T18:45:04,444 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T18:45:04,444 INFO [regionserver/d79ba0c344fb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T18:45:04,444 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T18:45:04,444 INFO [RS:1;d79ba0c344fb:43491 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43491 2024-11-22T18:45:04,446 DEBUG [RS:0;d79ba0c344fb:42191 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs 2024-11-22T18:45:04,446 INFO [RS:0;d79ba0c344fb:42191 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d79ba0c344fb%2C42191%2C1732301099646:(num 1732301101127) 2024-11-22T18:45:04,446 DEBUG [RS:0;d79ba0c344fb:42191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:04,446 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:45:04,446 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T18:45:04,446 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.ChoreService(370): Chore service for: regionserver/d79ba0c344fb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T18:45:04,446 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T18:45:04,446 INFO [regionserver/d79ba0c344fb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T18:45:04,447 INFO [RS:0;d79ba0c344fb:42191 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42191 2024-11-22T18:45:04,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d79ba0c344fb,43491,1732301099783 2024-11-22T18:45:04,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T18:45:04,450 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T18:45:04,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d79ba0c344fb,42191,1732301099646 2024-11-22T18:45:04,451 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T18:45:04,453 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d79ba0c344fb,43491,1732301099783] 2024-11-22T18:45:04,455 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d79ba0c344fb,43491,1732301099783 already deleted, retry=false 2024-11-22T18:45:04,456 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d79ba0c344fb,43491,1732301099783 expired; onlineServers=1 2024-11-22T18:45:04,456 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d79ba0c344fb,42191,1732301099646] 2024-11-22T18:45:04,457 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d79ba0c344fb,42191,1732301099646 already deleted, retry=false 2024-11-22T18:45:04,457 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d79ba0c344fb,42191,1732301099646 expired; onlineServers=0 2024-11-22T18:45:04,458 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd79ba0c344fb,42489,1732301098869' ***** 2024-11-22T18:45:04,458 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T18:45:04,458 INFO [M:0;d79ba0c344fb:42489 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T18:45:04,458 INFO [M:0;d79ba0c344fb:42489 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T18:45:04,458 DEBUG [M:0;d79ba0c344fb:42489 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T18:45:04,458 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T18:45:04,458 DEBUG [M:0;d79ba0c344fb:42489 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T18:45:04,458 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.small.0-1732301100789 {}] cleaner.HFileCleaner(306): Exit Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.small.0-1732301100789,5,FailOnTimeoutGroup] 2024-11-22T18:45:04,458 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.large.0-1732301100787 {}] cleaner.HFileCleaner(306): Exit Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.large.0-1732301100787,5,FailOnTimeoutGroup] 2024-11-22T18:45:04,459 INFO [M:0;d79ba0c344fb:42489 {}] hbase.ChoreService(370): Chore service for: master/d79ba0c344fb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T18:45:04,459 INFO [M:0;d79ba0c344fb:42489 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T18:45:04,459 DEBUG [M:0;d79ba0c344fb:42489 {}] master.HMaster(1795): Stopping service threads 2024-11-22T18:45:04,459 INFO [M:0;d79ba0c344fb:42489 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T18:45:04,459 INFO [M:0;d79ba0c344fb:42489 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T18:45:04,460 INFO [M:0;d79ba0c344fb:42489 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T18:45:04,460 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T18:45:04,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T18:45:04,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:04,461 DEBUG [M:0;d79ba0c344fb:42489 {}] zookeeper.ZKUtil(347): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T18:45:04,461 WARN [M:0;d79ba0c344fb:42489 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T18:45:04,461 INFO [M:0;d79ba0c344fb:42489 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/.lastflushedseqids 2024-11-22T18:45:04,473 INFO [M:0;d79ba0c344fb:42489 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T18:45:04,473 INFO [M:0;d79ba0c344fb:42489 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T18:45:04,474 DEBUG [M:0;d79ba0c344fb:42489 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T18:45:04,474 INFO [M:0;d79ba0c344fb:42489 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:04,474 DEBUG [M:0;d79ba0c344fb:42489 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:04,474 DEBUG [M:0;d79ba0c344fb:42489 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T18:45:04,474 DEBUG [M:0;d79ba0c344fb:42489 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:04,474 INFO [M:0;d79ba0c344fb:42489 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-22T18:45:04,486 DEBUG [M:0;d79ba0c344fb:42489 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8167bcddf2cb493c94e95e96465a7655 2024-11-22T18:45:04,501 DEBUG [M:0;d79ba0c344fb:42489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8167bcddf2cb493c94e95e96465a7655 is 82, key is hbase:meta,,1/info:regioninfo/1732301101568/Put/seqid=0 2024-11-22T18:45:04,502 INFO [M:0;d79ba0c344fb:42489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8167bcddf2cb493c94e95e96465a7655 2024-11-22T18:45:04,511 DEBUG [M:0;d79ba0c344fb:42489 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3ca9ad4d12c54aa7b600d5f0d04dc872 2024-11-22T18:45:04,525 DEBUG [M:0;d79ba0c344fb:42489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3ca9ad4d12c54aa7b600d5f0d04dc872 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732301102246/Put/seqid=0 2024-11-22T18:45:04,526 INFO [M:0;d79ba0c344fb:42489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3ca9ad4d12c54aa7b600d5f0d04dc872 2024-11-22T18:45:04,535 DEBUG [M:0;d79ba0c344fb:42489 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1ad8cfb0e9e642cab851d6d0bfe04adb 2024-11-22T18:45:04,548 DEBUG [M:0;d79ba0c344fb:42489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1ad8cfb0e9e642cab851d6d0bfe04adb is 69, key is d79ba0c344fb,42191,1732301099646/rs:state/1732301100889/Put/seqid=0 2024-11-22T18:45:04,548 INFO [M:0;d79ba0c344fb:42489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1ad8cfb0e9e642cab851d6d0bfe04adb 2024-11-22T18:45:04,551 DEBUG [M:0;d79ba0c344fb:42489 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8167bcddf2cb493c94e95e96465a7655 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8167bcddf2cb493c94e95e96465a7655 2024-11-22T18:45:04,552 INFO [M:0;d79ba0c344fb:42489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8167bcddf2cb493c94e95e96465a7655, entries=8, sequenceid=84, filesize=5.5 K 2024-11-22T18:45:04,553 DEBUG [M:0;d79ba0c344fb:42489 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3ca9ad4d12c54aa7b600d5f0d04dc872 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3ca9ad4d12c54aa7b600d5f0d04dc872 2024-11-22T18:45:04,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:04,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43491-0x10140fef8d70002, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:04,553 INFO [RS:1;d79ba0c344fb:43491 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T18:45:04,554 INFO [RS:1;d79ba0c344fb:43491 {}] regionserver.HRegionServer(1031): Exiting; stopping=d79ba0c344fb,43491,1732301099783; zookeeper connection closed. 2024-11-22T18:45:04,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:04,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42191-0x10140fef8d70001, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:04,554 INFO [RS:0;d79ba0c344fb:42191 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T18:45:04,554 INFO [RS:0;d79ba0c344fb:42191 {}] regionserver.HRegionServer(1031): Exiting; stopping=d79ba0c344fb,42191,1732301099646; zookeeper connection closed. 2024-11-22T18:45:04,554 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@412259ae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@412259ae 2024-11-22T18:45:04,554 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ed103f2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ed103f2 2024-11-22T18:45:04,554 INFO [M:0;d79ba0c344fb:42489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3ca9ad4d12c54aa7b600d5f0d04dc872, entries=9, sequenceid=84, filesize=6.5 K 2024-11-22T18:45:04,555 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-22T18:45:04,555 DEBUG [M:0;d79ba0c344fb:42489 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1ad8cfb0e9e642cab851d6d0bfe04adb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1ad8cfb0e9e642cab851d6d0bfe04adb 2024-11-22T18:45:04,556 INFO [M:0;d79ba0c344fb:42489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1ad8cfb0e9e642cab851d6d0bfe04adb, entries=2, sequenceid=84, filesize=5.1 K 2024-11-22T18:45:04,557 INFO [M:0;d79ba0c344fb:42489 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 83ms, sequenceid=84, compaction requested=false 2024-11-22T18:45:04,559 INFO [M:0;d79ba0c344fb:42489 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:04,559 DEBUG [M:0;d79ba0c344fb:42489 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732301104473Disabling compacts and flushes for region at 1732301104473Disabling writes for close at 1732301104474 (+1 ms)Obtaining lock to block concurrent updates at 1732301104474Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732301104474Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732301104475 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732301104475Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732301104475Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732301104500 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732301104500Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732301104504 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732301104525 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732301104525Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732301104528 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732301104547 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732301104547Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f46fe90: reopening flushed file at 1732301104550 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ae17a3d: reopening flushed file at 1732301104552 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68dc4653: reopening flushed file at 1732301104555 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 83ms, sequenceid=84, compaction requested=false at 1732301104557 (+2 ms)Writing region close event to WAL at 1732301104559 (+2 ms)Closed at 1732301104559 2024-11-22T18:45:04,561 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T18:45:04,561 INFO [M:0;d79ba0c344fb:42489 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T18:45:04,562 INFO [M:0;d79ba0c344fb:42489 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42489 2024-11-22T18:45:04,562 INFO [M:0;d79ba0c344fb:42489 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T18:45:04,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:04,664 INFO [M:0;d79ba0c344fb:42489 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T18:45:04,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x10140fef8d70000, quorum=127.0.0.1:56890, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:04,669 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T18:45:04,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T18:45:04,690 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:04,699 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: regionserver/d79ba0c344fb:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/d79ba0c344fb:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/d79ba0c344fb:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732301102626 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732301104157 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732301104324 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/d79ba0c344fb:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732301104327 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=967 (was 1016), ProcessCount=11 (was 11), AvailableMemoryMB=8372 (was 8730) 2024-11-22T18:45:04,705 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=967, ProcessCount=11, AvailableMemoryMB=8372 2024-11-22T18:45:04,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/hadoop-log-dir so I do NOT create it in target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150 2024-11-22T18:45:04,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/hadoop-tmp-dir so I do NOT create it in target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150 2024-11-22T18:45:04,706 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:04,709 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/cluster_1fe38f2f-c7ce-96d8-e89c-7b2e15e98c32, deleteOnExit=true 2024-11-22T18:45:04,713 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/cluster_1fe38f2f-c7ce-96d8-e89c-7b2e15e98c32/zookeeper_0, clientPort=56492, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/cluster_1fe38f2f-c7ce-96d8-e89c-7b2e15e98c32/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/cluster_1fe38f2f-c7ce-96d8-e89c-7b2e15e98c32/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T18:45:04,714 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56492 2024-11-22T18:45:04,726 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 with version=8 2024-11-22T18:45:04,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/hbase-staging 2024-11-22T18:45:04,730 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d79ba0c344fb:0 server-side Connection retries=45 2024-11-22T18:45:04,730 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:04,730 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:04,730 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T18:45:04,730 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:04,730 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T18:45:04,731 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T18:45:04,731 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T18:45:04,731 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35183 2024-11-22T18:45:04,733 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35183 connecting to ZooKeeper ensemble=127.0.0.1:56492 2024-11-22T18:45:04,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:351830x0, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T18:45:04,742 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35183-0x10140ff10d80000 connected 2024-11-22T18:45:04,759 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:04,759 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:04,760 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:45:04,760 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9, hbase.cluster.distributed=false 2024-11-22T18:45:04,762 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T18:45:04,764 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35183 2024-11-22T18:45:04,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35183 2024-11-22T18:45:04,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35183 2024-11-22T18:45:04,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35183 2024-11-22T18:45:04,766 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35183 2024-11-22T18:45:04,782 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d79ba0c344fb:0 server-side Connection retries=45 2024-11-22T18:45:04,782 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:04,782 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:04,782 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T18:45:04,782 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:04,782 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T18:45:04,782 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T18:45:04,782 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T18:45:04,783 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37551 2024-11-22T18:45:04,784 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37551 connecting to ZooKeeper ensemble=127.0.0.1:56492 2024-11-22T18:45:04,785 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:04,785 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:04,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:375510x0, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T18:45:04,788 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37551-0x10140ff10d80001 connected 2024-11-22T18:45:04,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:45:04,789 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T18:45:04,789 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.cache 2024-11-22T18:45:04,789 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T18:45:04,789 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T18:45:04,790 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T18:45:04,790 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T18:45:04,790 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T18:45:04,790 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T18:45:04,790 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T18:45:04,791 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T18:45:04,791 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.persistence is complete. 2024-11-22T18:45:04,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T18:45:04,792 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T18:45:04,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37551 2024-11-22T18:45:04,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37551 2024-11-22T18:45:04,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37551 2024-11-22T18:45:04,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37551 2024-11-22T18:45:04,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37551 2024-11-22T18:45:04,813 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d79ba0c344fb:0 server-side Connection retries=45 2024-11-22T18:45:04,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:04,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:04,813 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T18:45:04,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:04,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T18:45:04,813 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T18:45:04,813 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T18:45:04,814 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35781 2024-11-22T18:45:04,816 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35781 connecting to ZooKeeper ensemble=127.0.0.1:56492 2024-11-22T18:45:04,816 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:04,816 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:04,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:357810x0, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T18:45:04,820 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35781-0x10140ff10d80002 connected 2024-11-22T18:45:04,820 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:45:04,820 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T18:45:04,821 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.cache 2024-11-22T18:45:04,821 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T18:45:04,821 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T18:45:04,821 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T18:45:04,822 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T18:45:04,822 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T18:45:04,822 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T18:45:04,823 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T18:45:04,823 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.persistence is complete. 2024-11-22T18:45:04,823 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T18:45:04,824 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T18:45:04,825 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T18:45:04,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35781 2024-11-22T18:45:04,830 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35781 2024-11-22T18:45:04,830 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35781 2024-11-22T18:45:04,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35781 2024-11-22T18:45:04,833 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35781 2024-11-22T18:45:04,845 DEBUG [M:0;d79ba0c344fb:35183 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d79ba0c344fb:35183 2024-11-22T18:45:04,846 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:04,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:45:04,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:45:04,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:45:04,848 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:04,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T18:45:04,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T18:45:04,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:04,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:04,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:04,852 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T18:45:04,853 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d79ba0c344fb,35183,1732301104729 from backup master directory 2024-11-22T18:45:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:45:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:45:04,854 WARN [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T18:45:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:45:04,854 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:04,856 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/hbase.id] with ID: e4a78de2-13a1-41ca-b1c2-a471bb8d26cb 2024-11-22T18:45:04,856 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/.tmp/hbase.id 2024-11-22T18:45:04,860 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/hbase.id] 2024-11-22T18:45:04,898 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:04,899 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T18:45:04,900 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-22T18:45:04,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:04,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:04,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:04,907 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:45:04,909 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T18:45:04,909 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:45:04,931 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store 2024-11-22T18:45:04,938 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:04,938 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T18:45:04,938 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:04,938 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:04,938 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T18:45:04,938 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:04,938 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:04,938 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732301104938Disabling compacts and flushes for region at 1732301104938Disabling writes for close at 1732301104938Writing region close event to WAL at 1732301104938Closed at 1732301104938 2024-11-22T18:45:04,938 WARN [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/.initializing 2024-11-22T18:45:04,938 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/WALs/d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:04,945 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T18:45:04,949 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d79ba0c344fb%2C35183%2C1732301104729, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/WALs/d79ba0c344fb,35183,1732301104729, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/oldWALs, maxLogs=10 2024-11-22T18:45:04,969 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/WALs/d79ba0c344fb,35183,1732301104729/d79ba0c344fb%2C35183%2C1732301104729.1732301104950 2024-11-22T18:45:04,969 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T18:45:04,969 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:45:04,969 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:04,969 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,969 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,975 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T18:45:04,975 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:04,975 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:04,975 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,979 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T18:45:04,979 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:04,980 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:04,980 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,984 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T18:45:04,984 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:04,984 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:04,984 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,987 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T18:45:04,987 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:04,988 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:04,988 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,988 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,988 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,989 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,989 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,989 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T18:45:04,989 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:45:04,995 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:45:04,995 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71450622, jitterRate=0.06469723582267761}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T18:45:04,996 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732301104970Initializing all the Stores at 1732301104970Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301104970Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301104970Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301104970Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301104970Cleaning up temporary data from old regions at 1732301104989 (+19 ms)Region opened successfully at 1732301104996 (+7 ms) 2024-11-22T18:45:04,996 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T18:45:05,000 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@701ff3a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d79ba0c344fb/172.17.0.2:0 2024-11-22T18:45:05,002 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T18:45:05,002 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T18:45:05,002 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T18:45:05,002 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T18:45:05,002 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T18:45:05,003 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-22T18:45:05,003 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T18:45:05,005 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T18:45:05,006 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T18:45:05,008 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T18:45:05,008 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T18:45:05,009 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T18:45:05,010 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T18:45:05,010 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T18:45:05,011 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T18:45:05,012 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T18:45:05,013 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T18:45:05,015 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T18:45:05,018 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T18:45:05,019 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T18:45:05,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:05,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:05,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:05,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,022 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d79ba0c344fb,35183,1732301104729, sessionid=0x10140ff10d80000, setting cluster-up flag (Was=false) 2024-11-22T18:45:05,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,030 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T18:45:05,031 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:05,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,042 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T18:45:05,043 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:05,044 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T18:45:05,045 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T18:45:05,045 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T18:45:05,046 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T18:45:05,046 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d79ba0c344fb,35183,1732301104729 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T18:45:05,048 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:45:05,048 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:45:05,048 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:45:05,048 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:45:05,048 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d79ba0c344fb:0, corePoolSize=10, maxPoolSize=10 2024-11-22T18:45:05,048 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,048 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=2, maxPoolSize=2 2024-11-22T18:45:05,048 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,049 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732301135049 2024-11-22T18:45:05,049 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T18:45:05,049 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T18:45:05,049 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T18:45:05,050 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T18:45:05,050 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T18:45:05,050 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T18:45:05,050 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T18:45:05,050 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T18:45:05,050 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,050 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:05,050 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T18:45:05,050 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T18:45:05,051 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T18:45:05,050 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T18:45:05,051 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T18:45:05,051 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T18:45:05,051 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.large.0-1732301105051,5,FailOnTimeoutGroup] 2024-11-22T18:45:05,051 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.small.0-1732301105051,5,FailOnTimeoutGroup] 2024-11-22T18:45:05,051 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,052 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T18:45:05,052 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,052 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,064 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T18:45:05,064 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 2024-11-22T18:45:05,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:05,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T18:45:05,075 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T18:45:05,075 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:05,076 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:05,076 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T18:45:05,079 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T18:45:05,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:05,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:05,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T18:45:05,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T18:45:05,084 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:05,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:05,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T18:45:05,087 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T18:45:05,088 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:05,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:05,088 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T18:45:05,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740 2024-11-22T18:45:05,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740 2024-11-22T18:45:05,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T18:45:05,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T18:45:05,089 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T18:45:05,090 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T18:45:05,096 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:45:05,097 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73761348, jitterRate=0.09912973642349243}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T18:45:05,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732301105071Initializing all the Stores at 1732301105071Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301105071Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301105071Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301105071Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301105071Cleaning up temporary data from old regions at 1732301105089 (+18 ms)Region opened successfully at 1732301105097 (+8 ms) 2024-11-22T18:45:05,098 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T18:45:05,098 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T18:45:05,098 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T18:45:05,098 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T18:45:05,098 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T18:45:05,098 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T18:45:05,098 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732301105098Disabling compacts and flushes for region at 1732301105098Disabling writes for close at 1732301105098Writing region close event to WAL at 1732301105098Closed at 1732301105098 2024-11-22T18:45:05,099 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T18:45:05,099 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T18:45:05,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T18:45:05,101 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T18:45:05,102 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T18:45:05,135 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(746): ClusterId : e4a78de2-13a1-41ca-b1c2-a471bb8d26cb 2024-11-22T18:45:05,135 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(746): ClusterId : e4a78de2-13a1-41ca-b1c2-a471bb8d26cb 2024-11-22T18:45:05,135 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T18:45:05,135 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T18:45:05,138 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T18:45:05,138 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T18:45:05,138 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T18:45:05,138 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T18:45:05,140 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T18:45:05,140 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T18:45:05,141 DEBUG [RS:0;d79ba0c344fb:37551 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@741f25d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d79ba0c344fb/172.17.0.2:0 2024-11-22T18:45:05,141 DEBUG [RS:1;d79ba0c344fb:35781 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd0d5b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d79ba0c344fb/172.17.0.2:0 2024-11-22T18:45:05,153 DEBUG [RS:1;d79ba0c344fb:35781 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d79ba0c344fb:35781 2024-11-22T18:45:05,153 DEBUG [RS:0;d79ba0c344fb:37551 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d79ba0c344fb:37551 2024-11-22T18:45:05,154 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T18:45:05,154 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T18:45:05,154 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T18:45:05,154 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T18:45:05,154 DEBUG [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T18:45:05,154 DEBUG [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T18:45:05,155 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(2659): reportForDuty to master=d79ba0c344fb,35183,1732301104729 with port=37551, startcode=1732301104781 2024-11-22T18:45:05,155 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(2659): reportForDuty to master=d79ba0c344fb,35183,1732301104729 with port=35781, startcode=1732301104812 2024-11-22T18:45:05,155 DEBUG [RS:1;d79ba0c344fb:35781 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T18:45:05,155 DEBUG [RS:0;d79ba0c344fb:37551 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T18:45:05,158 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45291, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T18:45:05,158 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52533, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T18:45:05,158 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35183 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d79ba0c344fb,37551,1732301104781 2024-11-22T18:45:05,159 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35183 {}] master.ServerManager(517): Registering regionserver=d79ba0c344fb,37551,1732301104781 2024-11-22T18:45:05,160 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35183 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:05,160 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35183 {}] master.ServerManager(517): Registering regionserver=d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:05,160 DEBUG [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 2024-11-22T18:45:05,160 DEBUG [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T18:45:05,160 DEBUG [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T18:45:05,161 DEBUG [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 2024-11-22T18:45:05,161 DEBUG [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T18:45:05,161 DEBUG [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T18:45:05,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T18:45:05,168 DEBUG [RS:0;d79ba0c344fb:37551 {}] zookeeper.ZKUtil(111): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d79ba0c344fb,37551,1732301104781 2024-11-22T18:45:05,168 WARN [RS:0;d79ba0c344fb:37551 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T18:45:05,168 INFO [RS:0;d79ba0c344fb:37551 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:45:05,168 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d79ba0c344fb,37551,1732301104781] 2024-11-22T18:45:05,168 DEBUG [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,37551,1732301104781 2024-11-22T18:45:05,168 DEBUG [RS:1;d79ba0c344fb:35781 {}] zookeeper.ZKUtil(111): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:05,168 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d79ba0c344fb,35781,1732301104812] 2024-11-22T18:45:05,168 WARN [RS:1;d79ba0c344fb:35781 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T18:45:05,168 INFO [RS:1;d79ba0c344fb:35781 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:45:05,169 DEBUG [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:05,173 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T18:45:05,176 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T18:45:05,176 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T18:45:05,177 INFO [RS:0;d79ba0c344fb:37551 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T18:45:05,177 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,177 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T18:45:05,178 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T18:45:05,178 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,178 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,178 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,178 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,178 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,178 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,178 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d79ba0c344fb:0, corePoolSize=2, maxPoolSize=2 2024-11-22T18:45:05,178 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,178 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,178 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,179 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,179 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T18:45:05,179 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,179 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,179 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:45:05,179 DEBUG [RS:0;d79ba0c344fb:37551 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:45:05,179 INFO [RS:1;d79ba0c344fb:35781 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T18:45:05,179 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,179 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T18:45:05,179 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,180 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,180 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,180 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,180 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,180 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,37551,1732301104781-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T18:45:05,180 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T18:45:05,180 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,180 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,180 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,180 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,180 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,181 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,181 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d79ba0c344fb:0, corePoolSize=2, maxPoolSize=2 2024-11-22T18:45:05,181 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,181 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,181 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,181 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,181 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,181 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:05,181 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:45:05,181 DEBUG [RS:1;d79ba0c344fb:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:45:05,184 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,184 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,184 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,184 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,184 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,184 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,35781,1732301104812-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T18:45:05,195 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T18:45:05,196 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,37551,1732301104781-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,196 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,196 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.Replication(171): d79ba0c344fb,37551,1732301104781 started 2024-11-22T18:45:05,200 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T18:45:05,200 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,35781,1732301104812-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,200 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,200 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.Replication(171): d79ba0c344fb,35781,1732301104812 started 2024-11-22T18:45:05,210 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,210 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(1482): Serving as d79ba0c344fb,37551,1732301104781, RpcServer on d79ba0c344fb/172.17.0.2:37551, sessionid=0x10140ff10d80001 2024-11-22T18:45:05,210 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T18:45:05,210 DEBUG [RS:0;d79ba0c344fb:37551 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d79ba0c344fb,37551,1732301104781 2024-11-22T18:45:05,210 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,37551,1732301104781' 2024-11-22T18:45:05,210 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T18:45:05,211 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T18:45:05,212 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T18:45:05,212 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T18:45:05,212 DEBUG [RS:0;d79ba0c344fb:37551 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d79ba0c344fb,37551,1732301104781 2024-11-22T18:45:05,212 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,37551,1732301104781' 2024-11-22T18:45:05,212 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T18:45:05,212 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T18:45:05,213 DEBUG [RS:0;d79ba0c344fb:37551 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T18:45:05,213 INFO [RS:0;d79ba0c344fb:37551 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T18:45:05,213 INFO [RS:0;d79ba0c344fb:37551 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T18:45:05,218 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,218 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(1482): Serving as d79ba0c344fb,35781,1732301104812, RpcServer on d79ba0c344fb/172.17.0.2:35781, sessionid=0x10140ff10d80002 2024-11-22T18:45:05,218 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T18:45:05,218 DEBUG [RS:1;d79ba0c344fb:35781 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:05,218 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,35781,1732301104812' 2024-11-22T18:45:05,218 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T18:45:05,219 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T18:45:05,220 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T18:45:05,220 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T18:45:05,220 DEBUG [RS:1;d79ba0c344fb:35781 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:05,220 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,35781,1732301104812' 2024-11-22T18:45:05,220 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T18:45:05,220 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T18:45:05,221 DEBUG [RS:1;d79ba0c344fb:35781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T18:45:05,221 INFO [RS:1;d79ba0c344fb:35781 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T18:45:05,221 INFO [RS:1;d79ba0c344fb:35781 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T18:45:05,252 WARN [d79ba0c344fb:35183 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T18:45:05,313 INFO [RS:0;d79ba0c344fb:37551 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T18:45:05,314 INFO [RS:0;d79ba0c344fb:37551 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d79ba0c344fb%2C37551%2C1732301104781, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,37551,1732301104781, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs, maxLogs=55 2024-11-22T18:45:05,322 INFO [RS:1;d79ba0c344fb:35781 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T18:45:05,322 INFO [RS:1;d79ba0c344fb:35781 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d79ba0c344fb%2C35781%2C1732301104812, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,35781,1732301104812, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs, maxLogs=55 2024-11-22T18:45:05,334 INFO [RS:0;d79ba0c344fb:37551 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,37551,1732301104781/d79ba0c344fb%2C37551%2C1732301104781.1732301105316 2024-11-22T18:45:05,334 DEBUG [RS:0;d79ba0c344fb:37551 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T18:45:05,341 INFO [RS:1;d79ba0c344fb:35781 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,35781,1732301104812/d79ba0c344fb%2C35781%2C1732301104812.1732301105324 2024-11-22T18:45:05,341 DEBUG [RS:1;d79ba0c344fb:35781 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T18:45:05,502 DEBUG [d79ba0c344fb:35183 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-22T18:45:05,503 DEBUG [d79ba0c344fb:35183 {}] balancer.BalancerClusterState(204): Hosts are {d79ba0c344fb=0} racks are {/default-rack=0} 2024-11-22T18:45:05,503 DEBUG [d79ba0c344fb:35183 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T18:45:05,503 DEBUG [d79ba0c344fb:35183 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T18:45:05,503 DEBUG [d79ba0c344fb:35183 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T18:45:05,503 DEBUG [d79ba0c344fb:35183 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T18:45:05,503 INFO [d79ba0c344fb:35183 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T18:45:05,503 INFO [d79ba0c344fb:35183 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T18:45:05,503 DEBUG [d79ba0c344fb:35183 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T18:45:05,504 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:05,505 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d79ba0c344fb,35781,1732301104812, state=OPENING 2024-11-22T18:45:05,507 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T18:45:05,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:05,509 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T18:45:05,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:05,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:05,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:05,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d79ba0c344fb,35781,1732301104812}] 2024-11-22T18:45:05,663 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T18:45:05,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41289, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T18:45:05,668 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T18:45:05,668 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:45:05,668 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T18:45:05,669 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d79ba0c344fb%2C35781%2C1732301104812.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,35781,1732301104812, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs, maxLogs=55 2024-11-22T18:45:05,689 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,35781,1732301104812/d79ba0c344fb%2C35781%2C1732301104812.meta.1732301105670.meta 2024-11-22T18:45:05,689 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T18:45:05,689 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:45:05,690 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T18:45:05,690 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T18:45:05,690 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T18:45:05,690 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T18:45:05,690 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:05,690 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T18:45:05,690 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T18:45:05,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T18:45:05,692 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T18:45:05,692 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:05,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:05,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T18:45:05,693 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T18:45:05,693 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:05,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:05,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T18:45:05,694 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T18:45:05,694 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:05,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:05,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T18:45:05,695 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T18:45:05,695 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:05,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:45:05,695 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T18:45:05,696 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740 2024-11-22T18:45:05,696 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740 2024-11-22T18:45:05,696 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T18:45:05,696 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T18:45:05,696 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T18:45:05,697 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T18:45:05,698 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69940166, jitterRate=0.04218968749046326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T18:45:05,698 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T18:45:05,699 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732301105691Writing region info on filesystem at 1732301105691Initializing all the Stores at 1732301105691Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301105691Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301105691Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301105691Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301105691Cleaning up temporary data from old regions at 1732301105696 (+5 ms)Running coprocessor post-open hooks at 1732301105698 (+2 ms)Region opened successfully at 1732301105699 (+1 ms) 2024-11-22T18:45:05,699 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732301105662 2024-11-22T18:45:05,702 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T18:45:05,702 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T18:45:05,703 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:05,704 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d79ba0c344fb,35781,1732301104812, state=OPEN 2024-11-22T18:45:05,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T18:45:05,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T18:45:05,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T18:45:05,706 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:05,706 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:05,706 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:05,706 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:45:05,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T18:45:05,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d79ba0c344fb,35781,1732301104812 in 196 msec 2024-11-22T18:45:05,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T18:45:05,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 609 msec 2024-11-22T18:45:05,711 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T18:45:05,711 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T18:45:05,715 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:45:05,715 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,35781,1732301104812, seqNum=-1] 2024-11-22T18:45:05,716 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:45:05,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39291, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:45:05,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 678 msec 2024-11-22T18:45:05,724 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732301105724, completionTime=-1 2024-11-22T18:45:05,724 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-22T18:45:05,724 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T18:45:05,726 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T18:45:05,727 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732301165726 2024-11-22T18:45:05,727 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732301225727 2024-11-22T18:45:05,727 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-22T18:45:05,727 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,35183,1732301104729-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,727 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,35183,1732301104729-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,728 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,35183,1732301104729-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,728 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d79ba0c344fb:35183, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,728 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,728 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,730 DEBUG [master/d79ba0c344fb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T18:45:05,734 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.878sec 2024-11-22T18:45:05,734 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T18:45:05,734 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T18:45:05,734 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T18:45:05,734 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T18:45:05,734 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T18:45:05,734 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,35183,1732301104729-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T18:45:05,734 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,35183,1732301104729-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T18:45:05,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284befa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:45:05,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,35183,-1 for getting cluster id 2024-11-22T18:45:05,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:45:05,735 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T18:45:05,736 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T18:45:05,736 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,35183,1732301104729-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:05,737 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e4a78de2-13a1-41ca-b1c2-a471bb8d26cb' 2024-11-22T18:45:05,738 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:45:05,738 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e4a78de2-13a1-41ca-b1c2-a471bb8d26cb" 2024-11-22T18:45:05,738 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14005951, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:45:05,738 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,35183,-1] 2024-11-22T18:45:05,738 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:45:05,739 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:05,740 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:45:05,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2beb4aff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:45:05,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:45:05,743 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,35781,1732301104812, seqNum=-1] 2024-11-22T18:45:05,744 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:45:05,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:45:05,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:05,749 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T18:45:05,750 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:05,750 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f00b6a 2024-11-22T18:45:05,751 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T18:45:05,753 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T18:45:05,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35183 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732301105749', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:45:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35183 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732301105749 2024-11-22T18:45:05,756 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732301105749 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T18:45:05,756 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:05,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35183 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732301105749" procId is: 4 2024-11-22T18:45:05,757 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732301105749 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T18:45:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35183 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:45:05,769 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732301105749-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ba13a5284e412c1c9c91591759f80b67, NAME => 'testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732301105749', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 2024-11-22T18:45:05,776 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732301105749-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:05,776 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732301105749-pool-0 {}] regionserver.HRegion(1722): Closing ba13a5284e412c1c9c91591759f80b67, disabling compactions & flushes 2024-11-22T18:45:05,776 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732301105749-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:05,776 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732301105749-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:05,776 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732301105749-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. after waiting 0 ms 2024-11-22T18:45:05,776 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732301105749-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:05,776 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732301105749-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:05,776 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732301105749-pool-0 {}] regionserver.HRegion(1676): Region close journal for ba13a5284e412c1c9c91591759f80b67: Waiting for close lock at 1732301105776Disabling compacts and flushes for region at 1732301105776Disabling writes for close at 1732301105776Writing region close event to WAL at 1732301105776Closed at 1732301105776 2024-11-22T18:45:05,777 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732301105749 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T18:45:05,778 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732301105777"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732301105777"}]},"ts":"1732301105777"} 2024-11-22T18:45:05,780 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T18:45:05,781 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732301105749 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T18:45:05,782 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732301105749","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301105781"}]},"ts":"1732301105781"} 2024-11-22T18:45:05,784 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732301105749, state=ENABLING in hbase:meta 2024-11-22T18:45:05,784 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {d79ba0c344fb=0} racks are {/default-rack=0} 2024-11-22T18:45:05,785 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T18:45:05,785 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T18:45:05,785 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T18:45:05,785 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T18:45:05,785 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T18:45:05,785 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T18:45:05,785 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T18:45:05,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732301105749, region=ba13a5284e412c1c9c91591759f80b67, ASSIGN}] 2024-11-22T18:45:05,786 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732301105749, region=ba13a5284e412c1c9c91591759f80b67, ASSIGN 2024-11-22T18:45:05,787 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732301105749, region=ba13a5284e412c1c9c91591759f80b67, ASSIGN; state=OFFLINE, location=d79ba0c344fb,37551,1732301104781; forceNewPlan=false, retain=false 2024-11-22T18:45:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35183 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:45:05,937 INFO [d79ba0c344fb:35183 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T18:45:05,938 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ba13a5284e412c1c9c91591759f80b67, regionState=OPENING, regionLocation=d79ba0c344fb,37551,1732301104781 2024-11-22T18:45:05,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732301105749, region=ba13a5284e412c1c9c91591759f80b67, ASSIGN because future has completed 2024-11-22T18:45:05,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba13a5284e412c1c9c91591759f80b67, server=d79ba0c344fb,37551,1732301104781}] 2024-11-22T18:45:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35183 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:45:06,094 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T18:45:06,096 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56037, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T18:45:06,099 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:06,099 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ba13a5284e412c1c9c91591759f80b67, NAME => 'testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:45:06,099 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732301105749 ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,099 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:06,099 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,100 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,100 INFO [StoreOpener-ba13a5284e412c1c9c91591759f80b67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,105 INFO [StoreOpener-ba13a5284e412c1c9c91591759f80b67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba13a5284e412c1c9c91591759f80b67 columnFamilyName family 2024-11-22T18:45:06,105 DEBUG [StoreOpener-ba13a5284e412c1c9c91591759f80b67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:06,105 INFO [StoreOpener-ba13a5284e412c1c9c91591759f80b67-1 {}] regionserver.HStore(327): Store=ba13a5284e412c1c9c91591759f80b67/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:06,106 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,107 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,107 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,107 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,107 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,108 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,115 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:45:06,116 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ba13a5284e412c1c9c91591759f80b67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65569368, jitterRate=-0.02294027805328369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T18:45:06,116 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:06,117 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ba13a5284e412c1c9c91591759f80b67: Running coprocessor pre-open hook at 1732301106100Writing region info on filesystem at 1732301106100Initializing all the Stores at 1732301106100Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301106100Cleaning up temporary data from old regions at 1732301106107 (+7 ms)Running coprocessor post-open hooks at 1732301106116 (+9 ms)Region opened successfully at 1732301106117 (+1 ms) 2024-11-22T18:45:06,117 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67., pid=6, masterSystemTime=1732301106094 2024-11-22T18:45:06,121 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:06,121 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:06,121 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ba13a5284e412c1c9c91591759f80b67, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,37551,1732301104781 2024-11-22T18:45:06,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba13a5284e412c1c9c91591759f80b67, server=d79ba0c344fb,37551,1732301104781 because future has completed 2024-11-22T18:45:06,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T18:45:06,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ba13a5284e412c1c9c91591759f80b67, server=d79ba0c344fb,37551,1732301104781 in 183 msec 2024-11-22T18:45:06,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T18:45:06,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732301105749, region=ba13a5284e412c1c9c91591759f80b67, ASSIGN in 342 msec 2024-11-22T18:45:06,130 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732301105749 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T18:45:06,131 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732301105749","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301106130"}]},"ts":"1732301106130"} 2024-11-22T18:45:06,133 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732301105749, state=ENABLED in hbase:meta 2024-11-22T18:45:06,134 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732301105749 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T18:45:06,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732301105749 in 380 msec 2024-11-22T18:45:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35183 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:45:06,387 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732301105749 completed 2024-11-22T18:45:06,387 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732301105749 get assigned. Timeout = 60000ms 2024-11-22T18:45:06,388 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T18:45:06,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732301105749 assigned to meta. Checking AM states. 2024-11-22T18:45:06,391 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T18:45:06,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732301105749 assigned. 2024-11-22T18:45:06,395 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732301105749', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67., hostname=d79ba0c344fb,37551,1732301104781, seqNum=2] 2024-11-22T18:45:06,395 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:45:06,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:45:06,401 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ba13a5284e412c1c9c91591759f80b67 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T18:45:06,411 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d961df26c4f34c7aa0452dfaa3fc2933 2024-11-22T18:45:06,426 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/.tmp/family/d961df26c4f34c7aa0452dfaa3fc2933 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T18:45:06,426 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/.tmp/family/d961df26c4f34c7aa0452dfaa3fc2933 2024-11-22T18:45:06,429 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/.tmp/family/d961df26c4f34c7aa0452dfaa3fc2933 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933 2024-11-22T18:45:06,430 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933, delay=963 ms 2024-11-22T18:45:06,431 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T18:45:06,432 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for ba13a5284e412c1c9c91591759f80b67 in 31ms, sequenceid=6, compaction requested=false 2024-11-22T18:45:06,432 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732301105749' 2024-11-22T18:45:06,433 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ba13a5284e412c1c9c91591759f80b67: 2024-11-22T18:45:07,040 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T18:45:07,394 DEBUG [hfile-prefetch-1732301104157 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d961df26c4f34c7aa0452dfaa3fc2933_0. 2024-11-22T18:45:07,395 DEBUG [hfile-prefetch-1732301104157 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d961df26c4f34c7aa0452dfaa3fc2933_117. 2024-11-22T18:45:07,395 DEBUG [hfile-prefetch-1732301104157 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933, with total blocks 2, and data blocks 1 2024-11-22T18:45:07,395 DEBUG [hfile-prefetch-1732301104157 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T18:45:07,395 DEBUG [hfile-prefetch-1732301104157 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T18:45:07,395 DEBUG [hfile-prefetch-1732301104157 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933. Will sleep 100ms and try the verification again. 2024-11-22T18:45:07,424 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T18:45:07,424 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T18:45:07,424 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T18:45:07,424 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T18:45:07,428 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T18:45:07,428 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732301107424 2024-11-22T18:45:07,428 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T18:45:07,433 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;d79ba0c344fb:37551,5,FailOnTimeoutGroup] 2024-11-22T18:45:07,433 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd79ba0c344fb,37551,1732301104781' ***** 2024-11-22T18:45:07,434 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-22T18:45:07,434 INFO [RS:0;d79ba0c344fb:37551 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T18:45:07,434 INFO [RS:0;d79ba0c344fb:37551 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.persistence 2024-11-22T18:45:07,434 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T18:45:07,434 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T18:45:07,434 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T18:45:07,434 INFO [RS:0;d79ba0c344fb:37551 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T18:45:07,434 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T18:45:07,444 DEBUG [RS:0;d79ba0c344fb:37551 {}] bucket.BucketCache(1487): Thread RS:0;d79ba0c344fb:37551 started persisting bucket cache to file 2024-11-22T18:45:07,444 DEBUG [RS:0;d79ba0c344fb:37551 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T18:45:07,444 DEBUG [RS:0;d79ba0c344fb:37551 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T18:45:07,451 DEBUG [RS:0;d79ba0c344fb:37551 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T18:45:07,451 DEBUG [RS:0;d79ba0c344fb:37551 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732301107444 2024-11-22T18:45:07,452 DEBUG [RS:0;d79ba0c344fb:37551 {}] bucket.BucketCache(1509): Thread RS:0;d79ba0c344fb:37551 finished persisting bucket cache to file, renaming 2024-11-22T18:45:07,452 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T18:45:07,452 INFO [RS:0;d79ba0c344fb:37551 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T18:45:07,452 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T18:45:07,452 INFO [RS:0;d79ba0c344fb:37551 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T18:45:07,452 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(3091): Received CLOSE for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,452 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(959): stopping server d79ba0c344fb,37551,1732301104781 2024-11-22T18:45:07,452 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T18:45:07,453 INFO [RS:0;d79ba0c344fb:37551 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d79ba0c344fb:37551. 2024-11-22T18:45:07,453 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T18:45:07,453 DEBUG [RS:0;d79ba0c344fb:37551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:45:07,453 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ba13a5284e412c1c9c91591759f80b67, disabling compactions & flushes 2024-11-22T18:45:07,453 DEBUG [RS:0;d79ba0c344fb:37551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:07,453 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:07,453 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:07,453 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T18:45:07,453 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. after waiting 0 ms 2024-11-22T18:45:07,453 DEBUG [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(1325): Online Regions={ba13a5284e412c1c9c91591759f80b67=testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67.} 2024-11-22T18:45:07,453 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:07,453 DEBUG [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(1351): Waiting on ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,454 DEBUG [StoreFileCloser-ba13a5284e412c1c9c91591759f80b67-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933 2024-11-22T18:45:07,454 WARN [hfile-prefetch-1732301104157 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-22T18:45:07,454 DEBUG [hfile-prefetch-1732301104157 {}] hfile.PrefetchExecutor(121): Prefetch completed for d961df26c4f34c7aa0452dfaa3fc2933 2024-11-22T18:45:07,458 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T18:45:07,459 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:07,459 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ba13a5284e412c1c9c91591759f80b67: Waiting for close lock at 1732301107453Running coprocessor pre-close hooks at 1732301107453Disabling compacts and flushes for region at 1732301107453Disabling writes for close at 1732301107453Writing region close event to WAL at 1732301107454 (+1 ms)Running coprocessor post-close hooks at 1732301107459 (+5 ms)Closed at 1732301107459 2024-11-22T18:45:07,459 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:07,481 INFO [regionserver/d79ba0c344fb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:45:07,653 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(976): stopping server d79ba0c344fb,37551,1732301104781; all regions closed. 2024-11-22T18:45:07,655 DEBUG [RS:0;d79ba0c344fb:37551 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs 2024-11-22T18:45:07,655 INFO [RS:0;d79ba0c344fb:37551 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d79ba0c344fb%2C37551%2C1732301104781:(num 1732301105316) 2024-11-22T18:45:07,655 DEBUG [RS:0;d79ba0c344fb:37551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:07,655 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:45:07,655 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T18:45:07,656 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.ChoreService(370): Chore service for: regionserver/d79ba0c344fb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T18:45:07,656 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T18:45:07,656 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T18:45:07,656 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T18:45:07,656 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T18:45:07,656 INFO [regionserver/d79ba0c344fb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T18:45:07,656 INFO [RS:0;d79ba0c344fb:37551 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37551 2024-11-22T18:45:07,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T18:45:07,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d79ba0c344fb,37551,1732301104781 2024-11-22T18:45:07,660 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T18:45:07,662 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d79ba0c344fb,37551,1732301104781] 2024-11-22T18:45:07,665 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d79ba0c344fb,37551,1732301104781 already deleted, retry=false 2024-11-22T18:45:07,665 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of d79ba0c344fb,37551,1732301104781 on d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:07,669 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure d79ba0c344fb,37551,1732301104781, splitWal=true, meta=false 2024-11-22T18:45:07,671 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for d79ba0c344fb,37551,1732301104781 (carryingMeta=false) d79ba0c344fb,37551,1732301104781/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4de23ec1[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-22T18:45:07,671 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure d79ba0c344fb,37551,1732301104781, splitWal=true, meta=false 2024-11-22T18:45:07,673 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): d79ba0c344fb,37551,1732301104781 had 1 regions 2024-11-22T18:45:07,674 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure d79ba0c344fb,37551,1732301104781, splitWal=true, meta=false, isMeta: false 2024-11-22T18:45:07,675 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,37551,1732301104781-splitting 2024-11-22T18:45:07,675 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,37551,1732301104781-splitting dir is empty, no logs to split. 2024-11-22T18:45:07,675 INFO [PEWorker-2 {}] master.SplitWALManager(105): d79ba0c344fb,37551,1732301104781 WAL count=0, meta=false 2024-11-22T18:45:07,678 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,37551,1732301104781-splitting dir is empty, no logs to split. 2024-11-22T18:45:07,678 INFO [PEWorker-2 {}] master.SplitWALManager(105): d79ba0c344fb,37551,1732301104781 WAL count=0, meta=false 2024-11-22T18:45:07,678 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if d79ba0c344fb,37551,1732301104781 WAL splitting is done? wals=0, meta=false 2024-11-22T18:45:07,678 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for d79ba0c344fb,37551,1732301104781 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,37551,1732301104781-splitting does not exist 2024-11-22T18:45:07,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732301105749, region=ba13a5284e412c1c9c91591759f80b67, ASSIGN}] 2024-11-22T18:45:07,681 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732301105749, region=ba13a5284e412c1c9c91591759f80b67, ASSIGN 2024-11-22T18:45:07,682 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732301105749, region=ba13a5284e412c1c9c91591759f80b67, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T18:45:07,762 INFO [RS:0;d79ba0c344fb:37551 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T18:45:07,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:07,762 INFO [RS:0;d79ba0c344fb:37551 {}] regionserver.HRegionServer(1031): Exiting; stopping=d79ba0c344fb,37551,1732301104781; zookeeper connection closed. 2024-11-22T18:45:07,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37551-0x10140ff10d80001, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:07,763 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3b0ed25a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3b0ed25a 2024-11-22T18:45:07,833 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ba13a5284e412c1c9c91591759f80b67, regionState=OPENING, regionLocation=d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:07,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732301105749, region=ba13a5284e412c1c9c91591759f80b67, ASSIGN because future has completed 2024-11-22T18:45:07,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba13a5284e412c1c9c91591759f80b67, server=d79ba0c344fb,35781,1732301104812}] 2024-11-22T18:45:07,951 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d79ba0c344fb:0 server-side Connection retries=45 2024-11-22T18:45:07,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:07,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:07,951 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T18:45:07,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:45:07,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T18:45:07,951 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T18:45:07,951 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T18:45:07,952 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45573 2024-11-22T18:45:07,953 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45573 connecting to ZooKeeper ensemble=127.0.0.1:56492 2024-11-22T18:45:07,953 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:07,954 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:07,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:455730x0, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T18:45:07,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45573-0x10140ff10d80003 connected 2024-11-22T18:45:07,957 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45573-0x10140ff10d80003, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-22T18:45:07,957 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-22T18:45:07,958 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T18:45:07,959 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.cache 2024-11-22T18:45:07,959 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T18:45:07,959 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T18:45:07,959 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T18:45:07,960 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T18:45:07,960 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T18:45:07,961 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T18:45:07,961 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-22T18:45:07,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45573-0x10140ff10d80003, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T18:45:07,963 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-22T18:45:07,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45573-0x10140ff10d80003, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T18:45:07,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45573 2024-11-22T18:45:07,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45573 2024-11-22T18:45:07,968 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-22T18:45:07,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45573 2024-11-22T18:45:07,969 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T18:45:07,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45573 2024-11-22T18:45:07,969 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-22T18:45:07,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45573 2024-11-22T18:45:07,969 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.persistence is complete. 2024-11-22T18:45:07,971 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(746): ClusterId : e4a78de2-13a1-41ca-b1c2-a471bb8d26cb 2024-11-22T18:45:07,971 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T18:45:07,973 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T18:45:07,973 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T18:45:07,975 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T18:45:07,975 DEBUG [RS:2;d79ba0c344fb:45573 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d8d9e88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d79ba0c344fb/172.17.0.2:0 2024-11-22T18:45:07,987 DEBUG [RS:2;d79ba0c344fb:45573 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;d79ba0c344fb:45573 2024-11-22T18:45:07,987 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T18:45:07,987 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T18:45:07,987 DEBUG [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T18:45:07,988 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(2659): reportForDuty to master=d79ba0c344fb,35183,1732301104729 with port=45573, startcode=1732301107950 2024-11-22T18:45:07,988 DEBUG [RS:2;d79ba0c344fb:45573 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T18:45:07,990 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52153, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T18:45:07,990 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:07,991 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ba13a5284e412c1c9c91591759f80b67, NAME => 'testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:45:07,991 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35183 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d79ba0c344fb,45573,1732301107950 2024-11-22T18:45:07,991 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35183 {}] master.ServerManager(517): Registering regionserver=d79ba0c344fb,45573,1732301107950 2024-11-22T18:45:07,991 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732301105749 ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,991 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:45:07,991 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,992 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,992 INFO [StoreOpener-ba13a5284e412c1c9c91591759f80b67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,992 DEBUG [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9 2024-11-22T18:45:07,992 DEBUG [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T18:45:07,993 DEBUG [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T18:45:07,993 INFO [StoreOpener-ba13a5284e412c1c9c91591759f80b67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba13a5284e412c1c9c91591759f80b67 columnFamilyName family 2024-11-22T18:45:07,993 DEBUG [StoreOpener-ba13a5284e412c1c9c91591759f80b67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:45:07,995 DEBUG [StoreFileOpener-ba13a5284e412c1c9c91591759f80b67-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933, delay=943 ms 2024-11-22T18:45:07,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T18:45:07,996 DEBUG [RS:2;d79ba0c344fb:45573 {}] zookeeper.ZKUtil(111): regionserver:45573-0x10140ff10d80003, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d79ba0c344fb,45573,1732301107950 2024-11-22T18:45:07,996 WARN [RS:2;d79ba0c344fb:45573 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T18:45:07,996 INFO [RS:2;d79ba0c344fb:45573 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:45:07,996 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d79ba0c344fb,45573,1732301107950] 2024-11-22T18:45:07,996 DEBUG [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,45573,1732301107950 2024-11-22T18:45:07,996 DEBUG [StoreOpener-ba13a5284e412c1c9c91591759f80b67-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933 2024-11-22T18:45:07,996 INFO [StoreOpener-ba13a5284e412c1c9c91591759f80b67-1 {}] regionserver.HStore(327): Store=ba13a5284e412c1c9c91591759f80b67/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:45:07,997 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,997 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,998 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,998 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,998 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,999 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:07,999 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened ba13a5284e412c1c9c91591759f80b67; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65759115, jitterRate=-0.02011282742023468}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T18:45:08,000 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:08,001 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ba13a5284e412c1c9c91591759f80b67: Running coprocessor pre-open hook at 1732301107992Writing region info on filesystem at 1732301107992Initializing all the Stores at 1732301107992Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301107992Cleaning up temporary data from old regions at 1732301107998 (+6 ms)Running coprocessor post-open hooks at 1732301108000 (+2 ms)Region opened successfully at 1732301108001 (+1 ms) 2024-11-22T18:45:08,001 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67., pid=9, masterSystemTime=1732301107988 2024-11-22T18:45:08,002 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T18:45:08,004 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:08,004 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:08,004 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T18:45:08,004 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ba13a5284e412c1c9c91591759f80b67, regionState=OPEN, openSeqNum=10, regionLocation=d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:08,005 INFO [RS:2;d79ba0c344fb:45573 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T18:45:08,005 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:08,005 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T18:45:08,005 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T18:45:08,006 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d79ba0c344fb:0, corePoolSize=2, maxPoolSize=2 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:45:08,006 DEBUG [RS:2;d79ba0c344fb:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:45:08,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba13a5284e412c1c9c91591759f80b67, server=d79ba0c344fb,35781,1732301104812 because future has completed 2024-11-22T18:45:08,008 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:08,009 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:08,009 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:08,009 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:08,009 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:08,009 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,45573,1732301107950-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T18:45:08,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-22T18:45:08,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ba13a5284e412c1c9c91591759f80b67, server=d79ba0c344fb,35781,1732301104812 in 172 msec 2024-11-22T18:45:08,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T18:45:08,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732301105749, region=ba13a5284e412c1c9c91591759f80b67, ASSIGN in 331 msec 2024-11-22T18:45:08,013 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server d79ba0c344fb,37551,1732301104781 after splitting done 2024-11-22T18:45:08,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure d79ba0c344fb,37551,1732301104781, splitWal=true, meta=false in 347 msec 2024-11-22T18:45:08,024 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T18:45:08,024 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,45573,1732301107950-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:08,024 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:08,024 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.Replication(171): d79ba0c344fb,45573,1732301107950 started 2024-11-22T18:45:08,039 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:45:08,039 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(1482): Serving as d79ba0c344fb,45573,1732301107950, RpcServer on d79ba0c344fb/172.17.0.2:45573, sessionid=0x10140ff10d80003 2024-11-22T18:45:08,039 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T18:45:08,039 DEBUG [RS:2;d79ba0c344fb:45573 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d79ba0c344fb,45573,1732301107950 2024-11-22T18:45:08,039 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,45573,1732301107950' 2024-11-22T18:45:08,039 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T18:45:08,041 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T18:45:08,041 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T18:45:08,042 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T18:45:08,042 DEBUG [RS:2;d79ba0c344fb:45573 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d79ba0c344fb,45573,1732301107950 2024-11-22T18:45:08,042 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,45573,1732301107950' 2024-11-22T18:45:08,042 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T18:45:08,042 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T18:45:08,042 DEBUG [RS:2;d79ba0c344fb:45573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T18:45:08,043 INFO [RS:2;d79ba0c344fb:45573 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T18:45:08,043 INFO [RS:2;d79ba0c344fb:45573 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T18:45:08,143 INFO [RS:2;d79ba0c344fb:45573 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T18:45:08,144 INFO [RS:2;d79ba0c344fb:45573 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d79ba0c344fb%2C45573%2C1732301107950, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,45573,1732301107950, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs, maxLogs=55 2024-11-22T18:45:08,163 INFO [RS:2;d79ba0c344fb:45573 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/WALs/d79ba0c344fb,45573,1732301107950/d79ba0c344fb%2C45573%2C1732301107950.1732301108145 2024-11-22T18:45:08,163 DEBUG [RS:2;d79ba0c344fb:45573 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T18:45:08,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T18:45:08,539 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:45:08,540 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:45:08,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:08,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:08,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:45:08,540 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T18:45:08,540 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1549915307, stopped=false 2024-11-22T18:45:08,540 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d79ba0c344fb,35183,1732301104729 2024-11-22T18:45:08,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:08,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:08,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x10140ff10d80003, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T18:45:08,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:08,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:08,542 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T18:45:08,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x10140ff10d80003, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:08,543 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:45:08,543 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:45:08,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:08,543 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd79ba0c344fb,35781,1732301104812' ***** 2024-11-22T18:45:08,543 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T18:45:08,543 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45573-0x10140ff10d80003, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:45:08,543 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:45:08,543 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd79ba0c344fb,45573,1732301107950' ***** 2024-11-22T18:45:08,543 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T18:45:08,543 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:45:08,544 INFO [RS:2;d79ba0c344fb:45573 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T18:45:08,544 INFO [RS:2;d79ba0c344fb:45573 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.persistence 2024-11-22T18:45:08,544 INFO [RS:1;d79ba0c344fb:35781 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T18:45:08,544 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T18:45:08,545 INFO [RS:1;d79ba0c344fb:35781 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/681865c7-85b5-3ac9-14ea-d4ff1bfc7150/bucket.persistence 2024-11-22T18:45:08,544 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T18:45:08,545 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T18:45:08,545 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T18:45:08,545 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T18:45:08,545 INFO [RS:1;d79ba0c344fb:35781 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T18:45:08,545 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T18:45:08,545 INFO [RS:2;d79ba0c344fb:45573 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T18:45:08,545 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T18:45:08,546 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T18:45:08,555 DEBUG [RS:1;d79ba0c344fb:35781 {}] bucket.BucketCache(1487): Thread RS:1;d79ba0c344fb:35781 started persisting bucket cache to file 2024-11-22T18:45:08,556 DEBUG [RS:2;d79ba0c344fb:45573 {}] bucket.BucketCache(1487): Thread RS:2;d79ba0c344fb:45573 started persisting bucket cache to file 2024-11-22T18:45:08,556 DEBUG [RS:1;d79ba0c344fb:35781 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T18:45:08,556 DEBUG [RS:1;d79ba0c344fb:35781 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T18:45:08,556 DEBUG [RS:2;d79ba0c344fb:45573 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T18:45:08,556 DEBUG [RS:2;d79ba0c344fb:45573 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T18:45:08,559 DEBUG [RS:1;d79ba0c344fb:35781 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T18:45:08,559 DEBUG [RS:1;d79ba0c344fb:35781 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732301108555 2024-11-22T18:45:08,559 DEBUG [RS:1;d79ba0c344fb:35781 {}] bucket.BucketCache(1509): Thread RS:1;d79ba0c344fb:35781 finished persisting bucket cache to file, renaming 2024-11-22T18:45:08,559 DEBUG [RS:2;d79ba0c344fb:45573 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T18:45:08,559 DEBUG [RS:2;d79ba0c344fb:45573 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732301108556 2024-11-22T18:45:08,559 DEBUG [RS:2;d79ba0c344fb:45573 {}] bucket.BucketCache(1509): Thread RS:2;d79ba0c344fb:45573 finished persisting bucket cache to file, renaming 2024-11-22T18:45:08,560 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T18:45:08,560 INFO [RS:1;d79ba0c344fb:35781 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T18:45:08,560 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T18:45:08,560 INFO [RS:1;d79ba0c344fb:35781 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T18:45:08,560 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(3091): Received CLOSE for ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:08,560 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(959): stopping server d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:08,560 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T18:45:08,560 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T18:45:08,560 INFO [RS:1;d79ba0c344fb:35781 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d79ba0c344fb:35781. 2024-11-22T18:45:08,560 INFO [RS:2;d79ba0c344fb:45573 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T18:45:08,560 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T18:45:08,560 DEBUG [RS:1;d79ba0c344fb:35781 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:45:08,560 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T18:45:08,560 INFO [RS:2;d79ba0c344fb:45573 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T18:45:08,560 DEBUG [RS:1;d79ba0c344fb:35781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:08,560 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(959): stopping server d79ba0c344fb,45573,1732301107950 2024-11-22T18:45:08,560 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ba13a5284e412c1c9c91591759f80b67, disabling compactions & flushes 2024-11-22T18:45:08,560 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T18:45:08,560 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:08,560 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T18:45:08,561 INFO [RS:2;d79ba0c344fb:45573 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;d79ba0c344fb:45573. 2024-11-22T18:45:08,561 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:08,561 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T18:45:08,561 DEBUG [RS:2;d79ba0c344fb:45573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:45:08,561 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T18:45:08,561 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. after waiting 0 ms 2024-11-22T18:45:08,561 DEBUG [RS:2;d79ba0c344fb:45573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:08,561 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:08,561 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T18:45:08,561 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(976): stopping server d79ba0c344fb,45573,1732301107950; all regions closed. 2024-11-22T18:45:08,561 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-22T18:45:08,561 DEBUG [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ba13a5284e412c1c9c91591759f80b67=testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67.} 2024-11-22T18:45:08,561 DEBUG [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ba13a5284e412c1c9c91591759f80b67 2024-11-22T18:45:08,561 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T18:45:08,561 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T18:45:08,562 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T18:45:08,562 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T18:45:08,562 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T18:45:08,562 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T18:45:08,562 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-22T18:45:08,563 DEBUG [StoreFileCloser-ba13a5284e412c1c9c91591759f80b67-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/family/d961df26c4f34c7aa0452dfaa3fc2933 2024-11-22T18:45:08,564 DEBUG [RS:2;d79ba0c344fb:45573 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs 2024-11-22T18:45:08,564 INFO [RS:2;d79ba0c344fb:45573 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d79ba0c344fb%2C45573%2C1732301107950:(num 1732301108145) 2024-11-22T18:45:08,565 DEBUG [RS:2;d79ba0c344fb:45573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:08,565 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:45:08,565 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T18:45:08,565 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.ChoreService(370): Chore service for: regionserver/d79ba0c344fb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T18:45:08,565 INFO [regionserver/d79ba0c344fb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T18:45:08,566 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T18:45:08,566 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T18:45:08,566 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T18:45:08,566 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T18:45:08,566 INFO [RS:2;d79ba0c344fb:45573 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45573 2024-11-22T18:45:08,569 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/default/testBlockEvictionOnGracefulStop1732301105749/ba13a5284e412c1c9c91591759f80b67/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-22T18:45:08,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x10140ff10d80003, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d79ba0c344fb,45573,1732301107950 2024-11-22T18:45:08,569 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T18:45:08,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T18:45:08,569 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:08,569 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ba13a5284e412c1c9c91591759f80b67: Waiting for close lock at 1732301108560Running coprocessor pre-close hooks at 1732301108560Disabling compacts and flushes for region at 1732301108560Disabling writes for close at 1732301108561 (+1 ms)Writing region close event to WAL at 1732301108564 (+3 ms)Running coprocessor post-close hooks at 1732301108569 (+5 ms)Closed at 1732301108569 2024-11-22T18:45:08,570 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67. 2024-11-22T18:45:08,569 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f5edc477918@6ba28a4d rejected from java.util.concurrent.ThreadPoolExecutor@2ea0095d[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 3] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-22T18:45:08,570 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d79ba0c344fb,45573,1732301107950] 2024-11-22T18:45:08,572 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d79ba0c344fb,45573,1732301107950 already deleted, retry=false 2024-11-22T18:45:08,572 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d79ba0c344fb,45573,1732301107950 expired; onlineServers=1 2024-11-22T18:45:08,575 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4b1e94764f1b4eb6b2f26afeed87ce4f 2024-11-22T18:45:08,588 INFO [regionserver/d79ba0c344fb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:45:08,589 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/info/4b1e94764f1b4eb6b2f26afeed87ce4f is 209, key is testBlockEvictionOnGracefulStop1732301105749,,1732301105753.ba13a5284e412c1c9c91591759f80b67./info:regioninfo/1732301108004/Put/seqid=0 2024-11-22T18:45:08,590 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/info/4b1e94764f1b4eb6b2f26afeed87ce4f 2024-11-22T18:45:08,599 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f4fdda677e914e2bb033dfa887cb4de6 2024-11-22T18:45:08,609 INFO [regionserver/d79ba0c344fb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:45:08,611 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/ns/f4fdda677e914e2bb033dfa887cb4de6 is 43, key is default/ns:d/1732301105718/Put/seqid=0 2024-11-22T18:45:08,612 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/ns/f4fdda677e914e2bb033dfa887cb4de6 2024-11-22T18:45:08,620 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8ed43d928a564af49c4b61e191fbd25b 2024-11-22T18:45:08,634 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/table/8ed43d928a564af49c4b61e191fbd25b is 80, key is testBlockEvictionOnGracefulStop1732301105749/table:state/1732301106130/Put/seqid=0 2024-11-22T18:45:08,634 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/table/8ed43d928a564af49c4b61e191fbd25b 2024-11-22T18:45:08,637 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/info/4b1e94764f1b4eb6b2f26afeed87ce4f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/info/4b1e94764f1b4eb6b2f26afeed87ce4f 2024-11-22T18:45:08,638 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/info/4b1e94764f1b4eb6b2f26afeed87ce4f, delay=934 ms 2024-11-22T18:45:08,639 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/info/4b1e94764f1b4eb6b2f26afeed87ce4f, entries=14, sequenceid=13, filesize=7.5 K 2024-11-22T18:45:08,639 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/ns/f4fdda677e914e2bb033dfa887cb4de6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/ns/f4fdda677e914e2bb033dfa887cb4de6 2024-11-22T18:45:08,640 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/ns/f4fdda677e914e2bb033dfa887cb4de6, delay=974 ms 2024-11-22T18:45:08,641 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/ns/f4fdda677e914e2bb033dfa887cb4de6, entries=2, sequenceid=13, filesize=5.0 K 2024-11-22T18:45:08,642 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/.tmp/table/8ed43d928a564af49c4b61e191fbd25b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/table/8ed43d928a564af49c4b61e191fbd25b 2024-11-22T18:45:08,643 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/table/8ed43d928a564af49c4b61e191fbd25b, delay=954 ms 2024-11-22T18:45:08,644 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/table/8ed43d928a564af49c4b61e191fbd25b, entries=2, sequenceid=13, filesize=5.3 K 2024-11-22T18:45:08,645 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=13, compaction requested=false 2024-11-22T18:45:08,645 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/info/4b1e94764f1b4eb6b2f26afeed87ce4f 2024-11-22T18:45:08,646 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/ns/f4fdda677e914e2bb033dfa887cb4de6 2024-11-22T18:45:08,646 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/table/8ed43d928a564af49c4b61e191fbd25b 2024-11-22T18:45:08,651 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-22T18:45:08,652 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T18:45:08,652 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T18:45:08,652 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732301108561Running coprocessor pre-close hooks at 1732301108561Disabling compacts and flushes for region at 1732301108561Disabling writes for close at 1732301108562 (+1 ms)Obtaining lock to block concurrent updates at 1732301108562Preparing flush snapshotting stores in 1588230740 at 1732301108562Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732301108562Flushing stores of hbase:meta,,1.1588230740 at 1732301108562Flushing 1588230740/info: creating writer at 1732301108563 (+1 ms)Flushing 1588230740/info: appending metadata at 1732301108589 (+26 ms)Flushing 1588230740/info: closing flushed file at 1732301108589Flushing 1588230740/ns: creating writer at 1732301108592 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732301108611 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732301108611Flushing 1588230740/table: creating writer at 1732301108614 (+3 ms)Flushing 1588230740/table: appending metadata at 1732301108633 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732301108633Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@593c9f2d: reopening flushed file at 1732301108637 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e1c0026: reopening flushed file at 1732301108639 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ad3cff0: reopening flushed file at 1732301108642 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=13, compaction requested=false at 1732301108645 (+3 ms)Writing region close event to WAL at 1732301108647 (+2 ms)Running coprocessor post-close hooks at 1732301108652 (+5 ms)Closed at 1732301108652 2024-11-22T18:45:08,652 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T18:45:08,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x10140ff10d80003, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:08,671 INFO [RS:2;d79ba0c344fb:45573 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T18:45:08,671 INFO [RS:2;d79ba0c344fb:45573 {}] regionserver.HRegionServer(1031): Exiting; stopping=d79ba0c344fb,45573,1732301107950; zookeeper connection closed. 2024-11-22T18:45:08,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x10140ff10d80003, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:08,672 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1077b984 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1077b984 2024-11-22T18:45:08,762 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(976): stopping server d79ba0c344fb,35781,1732301104812; all regions closed. 2024-11-22T18:45:08,764 DEBUG [RS:1;d79ba0c344fb:35781 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs 2024-11-22T18:45:08,764 INFO [RS:1;d79ba0c344fb:35781 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d79ba0c344fb%2C35781%2C1732301104812.meta:.meta(num 1732301105670) 2024-11-22T18:45:08,765 DEBUG [RS:1;d79ba0c344fb:35781 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/oldWALs 2024-11-22T18:45:08,765 INFO [RS:1;d79ba0c344fb:35781 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d79ba0c344fb%2C35781%2C1732301104812:(num 1732301105324) 2024-11-22T18:45:08,765 DEBUG [RS:1;d79ba0c344fb:35781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:45:08,765 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:45:08,765 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T18:45:08,766 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.ChoreService(370): Chore service for: regionserver/d79ba0c344fb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T18:45:08,766 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T18:45:08,766 INFO [regionserver/d79ba0c344fb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T18:45:08,766 INFO [RS:1;d79ba0c344fb:35781 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35781 2024-11-22T18:45:08,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d79ba0c344fb,35781,1732301104812 2024-11-22T18:45:08,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T18:45:08,768 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T18:45:08,770 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d79ba0c344fb,35781,1732301104812] 2024-11-22T18:45:08,772 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d79ba0c344fb,35781,1732301104812 already deleted, retry=false 2024-11-22T18:45:08,772 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d79ba0c344fb,35781,1732301104812 expired; onlineServers=0 2024-11-22T18:45:08,772 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd79ba0c344fb,35183,1732301104729' ***** 2024-11-22T18:45:08,772 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T18:45:08,772 INFO [M:0;d79ba0c344fb:35183 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T18:45:08,772 INFO [M:0;d79ba0c344fb:35183 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T18:45:08,772 DEBUG [M:0;d79ba0c344fb:35183 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T18:45:08,772 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T18:45:08,772 DEBUG [M:0;d79ba0c344fb:35183 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T18:45:08,772 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.large.0-1732301105051 {}] cleaner.HFileCleaner(306): Exit Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.large.0-1732301105051,5,FailOnTimeoutGroup] 2024-11-22T18:45:08,772 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.small.0-1732301105051 {}] cleaner.HFileCleaner(306): Exit Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.small.0-1732301105051,5,FailOnTimeoutGroup] 2024-11-22T18:45:08,773 INFO [M:0;d79ba0c344fb:35183 {}] hbase.ChoreService(370): Chore service for: master/d79ba0c344fb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T18:45:08,773 INFO [M:0;d79ba0c344fb:35183 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T18:45:08,773 DEBUG [M:0;d79ba0c344fb:35183 {}] master.HMaster(1795): Stopping service threads 2024-11-22T18:45:08,773 INFO [M:0;d79ba0c344fb:35183 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T18:45:08,773 INFO [M:0;d79ba0c344fb:35183 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T18:45:08,773 INFO [M:0;d79ba0c344fb:35183 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T18:45:08,773 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T18:45:08,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T18:45:08,774 DEBUG [M:0;d79ba0c344fb:35183 {}] zookeeper.ZKUtil(347): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T18:45:08,774 WARN [M:0;d79ba0c344fb:35183 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T18:45:08,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:45:08,775 INFO [M:0;d79ba0c344fb:35183 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/.lastflushedseqids 2024-11-22T18:45:08,781 INFO [M:0;d79ba0c344fb:35183 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T18:45:08,781 INFO [M:0;d79ba0c344fb:35183 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T18:45:08,781 DEBUG [M:0;d79ba0c344fb:35183 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T18:45:08,782 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:08,782 DEBUG [M:0;d79ba0c344fb:35183 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:08,782 DEBUG [M:0;d79ba0c344fb:35183 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T18:45:08,782 DEBUG [M:0;d79ba0c344fb:35183 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:08,782 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2024-11-22T18:45:08,791 DEBUG [M:0;d79ba0c344fb:35183 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4d6afe8dda0c4be3921b0126f85ac7af 2024-11-22T18:45:08,805 DEBUG [M:0;d79ba0c344fb:35183 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4d6afe8dda0c4be3921b0126f85ac7af is 82, key is hbase:meta,,1/info:regioninfo/1732301105702/Put/seqid=0 2024-11-22T18:45:08,805 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4d6afe8dda0c4be3921b0126f85ac7af 2024-11-22T18:45:08,813 DEBUG [M:0;d79ba0c344fb:35183 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 764bc54ad06440b39dc7e825401daf33 2024-11-22T18:45:08,827 DEBUG [M:0;d79ba0c344fb:35183 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/764bc54ad06440b39dc7e825401daf33 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732301106135/Put/seqid=0 2024-11-22T18:45:08,828 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/764bc54ad06440b39dc7e825401daf33 2024-11-22T18:45:08,836 DEBUG [M:0;d79ba0c344fb:35183 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ae7030ace9f141009cf50afc9d06eecf 2024-11-22T18:45:08,850 DEBUG [M:0;d79ba0c344fb:35183 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ae7030ace9f141009cf50afc9d06eecf is 69, key is d79ba0c344fb,35781,1732301104812/rs:state/1732301105160/Put/seqid=0 2024-11-22T18:45:08,851 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ae7030ace9f141009cf50afc9d06eecf 2024-11-22T18:45:08,853 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ae7030ace9f141009cf50afc9d06eecf 2024-11-22T18:45:08,854 DEBUG [M:0;d79ba0c344fb:35183 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4d6afe8dda0c4be3921b0126f85ac7af as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4d6afe8dda0c4be3921b0126f85ac7af 2024-11-22T18:45:08,855 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4d6afe8dda0c4be3921b0126f85ac7af, entries=8, sequenceid=83, filesize=5.5 K 2024-11-22T18:45:08,856 DEBUG [M:0;d79ba0c344fb:35183 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/764bc54ad06440b39dc7e825401daf33 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/764bc54ad06440b39dc7e825401daf33 2024-11-22T18:45:08,858 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/764bc54ad06440b39dc7e825401daf33, entries=9, sequenceid=83, filesize=6.4 K 2024-11-22T18:45:08,858 DEBUG [M:0;d79ba0c344fb:35183 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ae7030ace9f141009cf50afc9d06eecf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ae7030ace9f141009cf50afc9d06eecf 2024-11-22T18:45:08,860 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ae7030ace9f141009cf50afc9d06eecf 2024-11-22T18:45:08,860 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b711993-f7a5-42b4-4ab3-009845ddb73d/8c8addb1-4226-eee1-d58c-47edb43ef8f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ae7030ace9f141009cf50afc9d06eecf, entries=3, sequenceid=83, filesize=5.3 K 2024-11-22T18:45:08,860 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false 2024-11-22T18:45:08,862 INFO [M:0;d79ba0c344fb:35183 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:45:08,862 DEBUG [M:0;d79ba0c344fb:35183 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732301108781Disabling compacts and flushes for region at 1732301108781Disabling writes for close at 1732301108782 (+1 ms)Obtaining lock to block concurrent updates at 1732301108782Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732301108782Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1732301108782Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732301108782Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732301108783 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732301108804 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732301108804Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732301108807 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732301108827 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732301108827Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732301108830 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732301108850 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732301108850Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22477825: reopening flushed file at 1732301108853 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a8cbb9a: reopening flushed file at 1732301108856 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72b469ad: reopening flushed file at 1732301108858 (+2 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false at 1732301108861 (+3 ms)Writing region close event to WAL at 1732301108862 (+1 ms)Closed at 1732301108862 2024-11-22T18:45:08,863 INFO [M:0;d79ba0c344fb:35183 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T18:45:08,863 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T18:45:08,864 INFO [M:0;d79ba0c344fb:35183 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35183 2024-11-22T18:45:08,864 INFO [M:0;d79ba0c344fb:35183 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T18:45:08,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:08,871 INFO [RS:1;d79ba0c344fb:35781 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T18:45:08,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x10140ff10d80002, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:08,871 INFO [RS:1;d79ba0c344fb:35781 {}] regionserver.HRegionServer(1031): Exiting; stopping=d79ba0c344fb,35781,1732301104812; zookeeper connection closed. 2024-11-22T18:45:08,871 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47235086 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47235086 2024-11-22T18:45:08,871 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-22T18:45:08,966 INFO [M:0;d79ba0c344fb:35183 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T18:45:08,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:08,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35183-0x10140ff10d80000, quorum=127.0.0.1:56492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:45:08,969 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T18:45:08,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T18:45:08,970 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:45:08,977 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=897 (was 967), ProcessCount=11 (was 11), AvailableMemoryMB=8303 (was 8372)