2024-11-21 03:54:55,019 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-21 03:54:55,037 main DEBUG Took 0.016276 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 03:54:55,037 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 03:54:55,038 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 03:54:55,039 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 03:54:55,041 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,051 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 03:54:55,063 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,064 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,065 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,065 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,066 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,066 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,067 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,068 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,069 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,069 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,070 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,070 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,071 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,071 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,072 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,072 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,072 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,073 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,073 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,074 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,074 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:54:55,074 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,075 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 03:54:55,076 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:54:55,077 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 03:54:55,079 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 03:54:55,079 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 03:54:55,080 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 03:54:55,081 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 03:54:55,089 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 03:54:55,092 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 03:54:55,094 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 03:54:55,094 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 03:54:55,094 main DEBUG createAppenders(={Console}) 2024-11-21 03:54:55,095 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-21 03:54:55,096 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-21 03:54:55,096 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-21 03:54:55,096 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 03:54:55,097 main DEBUG OutputStream closed 2024-11-21 03:54:55,097 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 03:54:55,097 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 03:54:55,098 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-21 03:54:55,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 03:54:55,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 03:54:55,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 03:54:55,174 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 03:54:55,174 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 03:54:55,175 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 03:54:55,175 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 03:54:55,175 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 03:54:55,176 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 03:54:55,176 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 03:54:55,176 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 03:54:55,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 03:54:55,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 03:54:55,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 03:54:55,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 03:54:55,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 03:54:55,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 03:54:55,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 03:54:55,181 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 03:54:55,181 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-21 03:54:55,182 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 03:54:55,182 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-21T03:54:55,195 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-21 03:54:55,197 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 03:54:55,197 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T03:54:55,430 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2 2024-11-21T03:54:55,453 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=388, ProcessCount=11, AvailableMemoryMB=7187 2024-11-21T03:54:55,545 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T03:54:55,638 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:54:55,654 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/cluster_9c4a480f-50d6-6125-c59f-c3e0d27c1fbc, deleteOnExit=true 2024-11-21T03:54:55,806 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/cluster_9c4a480f-50d6-6125-c59f-c3e0d27c1fbc/zookeeper_0, clientPort=62150, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/cluster_9c4a480f-50d6-6125-c59f-c3e0d27c1fbc/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/cluster_9c4a480f-50d6-6125-c59f-c3e0d27c1fbc/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T03:54:55,817 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62150 2024-11-21T03:54:55,818 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:54:55,893 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 with version=8 2024-11-21T03:54:55,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/hbase-staging 2024-11-21T03:54:55,980 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T03:54:56,245 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fdc0a393186c:0 server-side Connection retries=45 2024-11-21T03:54:56,254 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:54:56,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:54:56,265 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:54:56,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:54:56,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:54:56,406 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T03:54:56,498 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:54:56,535 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T03:54:56,545 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T03:54:56,549 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:54:56,584 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 96900 (auto-detected) 2024-11-21T03:54:56,585 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T03:54:56,612 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42431 2024-11-21T03:54:56,634 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42431 connecting to ZooKeeper ensemble=127.0.0.1:62150 2024-11-21T03:54:56,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:424310x0, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:54:56,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42431-0x1015cd1686c0000 connected 2024-11-21T03:54:56,849 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:54:56,851 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:54:56,859 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:54:56,863 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1, hbase.cluster.distributed=false 2024-11-21T03:54:56,885 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:54:56,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42431 2024-11-21T03:54:56,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42431 2024-11-21T03:54:56,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42431 2024-11-21T03:54:56,892 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42431 2024-11-21T03:54:56,892 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42431 2024-11-21T03:54:56,966 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fdc0a393186c:0 server-side Connection retries=45 2024-11-21T03:54:56,967 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:54:56,968 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:54:56,968 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:54:56,968 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:54:56,969 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:54:56,971 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T03:54:56,974 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:54:56,975 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37035 2024-11-21T03:54:56,977 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37035 connecting to ZooKeeper ensemble=127.0.0.1:62150 2024-11-21T03:54:56,977 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:54:56,978 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:54:57,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370350x0, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:54:57,005 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37035-0x1015cd1686c0001 connected 2024-11-21T03:54:57,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:54:57,014 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T03:54:57,025 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/bucket.cache 2024-11-21T03:54:57,030 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T03:54:57,031 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T03:54:57,033 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T03:54:57,035 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T03:54:57,035 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T03:54:57,040 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T03:54:57,046 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T03:54:57,047 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T03:54:57,047 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/bucket.persistence is complete. 2024-11-21T03:54:57,049 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T03:54:57,055 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:54:57,058 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37035 2024-11-21T03:54:57,059 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37035 2024-11-21T03:54:57,060 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37035 2024-11-21T03:54:57,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37035 2024-11-21T03:54:57,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37035 2024-11-21T03:54:57,087 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fdc0a393186c:0 server-side Connection retries=45 2024-11-21T03:54:57,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:54:57,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:54:57,088 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:54:57,089 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:54:57,089 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:54:57,089 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T03:54:57,090 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:54:57,091 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46083 2024-11-21T03:54:57,094 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46083 connecting to ZooKeeper ensemble=127.0.0.1:62150 2024-11-21T03:54:57,094 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:54:57,095 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:54:57,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460830x0, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:54:57,128 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:460830x0, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:54:57,128 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46083-0x1015cd1686c0002 connected 2024-11-21T03:54:57,129 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T03:54:57,130 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/bucket.cache 2024-11-21T03:54:57,131 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T03:54:57,131 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T03:54:57,132 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T03:54:57,135 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T03:54:57,135 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T03:54:57,135 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T03:54:57,136 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T03:54:57,136 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T03:54:57,136 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/bucket.persistence is complete. 2024-11-21T03:54:57,138 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T03:54:57,140 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:54:57,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46083 2024-11-21T03:54:57,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46083 2024-11-21T03:54:57,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46083 2024-11-21T03:54:57,151 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46083 2024-11-21T03:54:57,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46083 2024-11-21T03:54:57,171 DEBUG [M:0;fdc0a393186c:42431 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fdc0a393186c:42431 2024-11-21T03:54:57,173 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fdc0a393186c,42431,1732161296085 2024-11-21T03:54:57,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:54:57,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:54:57,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:54:57,209 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fdc0a393186c,42431,1732161296085 2024-11-21T03:54:57,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T03:54:57,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T03:54:57,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:57,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:57,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:57,240 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T03:54:57,241 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fdc0a393186c,42431,1732161296085 from backup master directory 2024-11-21T03:54:57,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:54:57,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fdc0a393186c,42431,1732161296085 2024-11-21T03:54:57,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:54:57,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:54:57,253 WARN [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:54:57,254 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fdc0a393186c,42431,1732161296085 2024-11-21T03:54:57,256 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T03:54:57,258 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T03:54:57,268 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/hbase.id] with ID: 4d3b120e-c749-4c90-8e40-a29ac6eae3be 2024-11-21T03:54:57,269 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/.tmp/hbase.id 2024-11-21T03:54:57,275 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/hbase.id] 2024-11-21T03:54:57,315 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:54:57,321 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T03:54:57,349 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-21T03:54:57,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:57,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:57,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:57,404 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:54:57,408 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T03:54:57,449 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:54:57,455 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:54:57,510 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store 2024-11-21T03:54:57,531 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T03:54:57,536 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:54:57,538 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T03:54:57,539 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:54:57,539 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:54:57,542 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T03:54:57,542 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:54:57,542 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:54:57,543 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732161297538Disabling compacts and flushes for region at 1732161297538Disabling writes for close at 1732161297542 (+4 ms)Writing region close event to WAL at 1732161297542Closed at 1732161297542 2024-11-21T03:54:57,544 WARN [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/.initializing 2024-11-21T03:54:57,544 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/WALs/fdc0a393186c,42431,1732161296085 2024-11-21T03:54:57,557 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:54:57,573 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fdc0a393186c%2C42431%2C1732161296085, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/WALs/fdc0a393186c,42431,1732161296085, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/oldWALs, maxLogs=10 2024-11-21T03:54:57,636 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/WALs/fdc0a393186c,42431,1732161296085/fdc0a393186c%2C42431%2C1732161296085.1732161297579 2024-11-21T03:54:57,637 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T03:54:57,637 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:54:57,638 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:54:57,643 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,645 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,716 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T03:54:57,720 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:57,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:54:57,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,729 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T03:54:57,729 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:57,730 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:54:57,730 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,735 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T03:54:57,735 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:57,736 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:54:57,736 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T03:54:57,741 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:57,742 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:54:57,742 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,747 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,747 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,751 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,752 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,756 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T03:54:57,758 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:54:57,765 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:54:57,766 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62970332, jitterRate=-0.06166893243789673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T03:54:57,776 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732161297661Initializing all the Stores at 1732161297662 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161297663 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161297664 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161297664Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161297664Cleaning up temporary data from old regions at 1732161297752 (+88 ms)Region opened successfully at 1732161297776 (+24 ms) 2024-11-21T03:54:57,778 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T03:54:57,815 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c34e25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fdc0a393186c/172.17.0.2:0 2024-11-21T03:54:57,854 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T03:54:57,867 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T03:54:57,867 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T03:54:57,870 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T03:54:57,871 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T03:54:57,877 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-21T03:54:57,878 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T03:54:57,904 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T03:54:57,913 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T03:54:57,960 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T03:54:57,963 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T03:54:57,965 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T03:54:57,976 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T03:54:57,979 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T03:54:57,983 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T03:54:57,993 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T03:54:57,996 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T03:54:58,004 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T03:54:58,026 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T03:54:58,037 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T03:54:58,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T03:54:58,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T03:54:58,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T03:54:58,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,064 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fdc0a393186c,42431,1732161296085, sessionid=0x1015cd1686c0000, setting cluster-up flag (Was=false) 2024-11-21T03:54:58,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,112 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T03:54:58,116 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fdc0a393186c,42431,1732161296085 2024-11-21T03:54:58,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,171 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T03:54:58,173 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fdc0a393186c,42431,1732161296085 2024-11-21T03:54:58,180 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T03:54:58,253 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T03:54:58,260 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(746): ClusterId : 4d3b120e-c749-4c90-8e40-a29ac6eae3be 2024-11-21T03:54:58,260 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(746): ClusterId : 4d3b120e-c749-4c90-8e40-a29ac6eae3be 2024-11-21T03:54:58,262 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T03:54:58,262 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T03:54:58,264 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T03:54:58,271 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T03:54:58,275 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fdc0a393186c,42431,1732161296085 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T03:54:58,298 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T03:54:58,298 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T03:54:58,298 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T03:54:58,298 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T03:54:58,299 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:54:58,300 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:54:58,300 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:54:58,300 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:54:58,300 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fdc0a393186c:0, corePoolSize=10, maxPoolSize=10 2024-11-21T03:54:58,300 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,301 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fdc0a393186c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:54:58,301 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,303 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732161328303 2024-11-21T03:54:58,304 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T03:54:58,306 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T03:54:58,306 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:54:58,307 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T03:54:58,310 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T03:54:58,310 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T03:54:58,311 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T03:54:58,311 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T03:54:58,312 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:58,312 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T03:54:58,313 DEBUG [RS:0;fdc0a393186c:37035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13cf155c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fdc0a393186c/172.17.0.2:0 2024-11-21T03:54:58,312 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T03:54:58,312 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,316 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T03:54:58,318 DEBUG [RS:1;fdc0a393186c:46083 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d0e401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fdc0a393186c/172.17.0.2:0 2024-11-21T03:54:58,318 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T03:54:58,319 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T03:54:58,320 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T03:54:58,323 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T03:54:58,323 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T03:54:58,327 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.large.0-1732161298325,5,FailOnTimeoutGroup] 2024-11-21T03:54:58,327 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.small.0-1732161298327,5,FailOnTimeoutGroup] 2024-11-21T03:54:58,328 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,328 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T03:54:58,329 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,330 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,335 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T03:54:58,336 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 2024-11-21T03:54:58,339 DEBUG [RS:1;fdc0a393186c:46083 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fdc0a393186c:46083 2024-11-21T03:54:58,339 DEBUG [RS:0;fdc0a393186c:37035 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fdc0a393186c:37035 2024-11-21T03:54:58,343 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T03:54:58,343 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T03:54:58,344 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T03:54:58,344 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T03:54:58,344 DEBUG [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T03:54:58,344 DEBUG [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T03:54:58,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:54:58,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T03:54:58,347 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(2659): reportForDuty to master=fdc0a393186c,42431,1732161296085 with port=37035, startcode=1732161296933 2024-11-21T03:54:58,347 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(2659): reportForDuty to master=fdc0a393186c,42431,1732161296085 with port=46083, startcode=1732161297087 2024-11-21T03:54:58,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T03:54:58,351 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:58,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:54:58,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T03:54:58,356 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T03:54:58,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:58,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:54:58,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T03:54:58,360 DEBUG [RS:1;fdc0a393186c:46083 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T03:54:58,360 DEBUG [RS:0;fdc0a393186c:37035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T03:54:58,362 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T03:54:58,362 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:58,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:54:58,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T03:54:58,366 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T03:54:58,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:58,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:54:58,367 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T03:54:58,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740 2024-11-21T03:54:58,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740 2024-11-21T03:54:58,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T03:54:58,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T03:54:58,369 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T03:54:58,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T03:54:58,376 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:54:58,376 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74530483, jitterRate=0.11059074103832245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T03:54:58,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732161298344Initializing all the Stores at 1732161298345 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161298345Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161298345Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161298345Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161298345Cleaning up temporary data from old regions at 1732161298369 (+24 ms)Region opened successfully at 1732161298380 (+11 ms) 2024-11-21T03:54:58,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T03:54:58,381 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T03:54:58,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T03:54:58,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T03:54:58,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T03:54:58,382 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T03:54:58,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732161298381Disabling compacts and flushes for region at 1732161298381Disabling writes for close at 1732161298381Writing region close event to WAL at 1732161298382 (+1 ms)Closed at 1732161298382 2024-11-21T03:54:58,385 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:54:58,385 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T03:54:58,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T03:54:58,399 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T03:54:58,402 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T03:54:58,430 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34119, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T03:54:58,430 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T03:54:58,438 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42431 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fdc0a393186c,46083,1732161297087 2024-11-21T03:54:58,441 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42431 {}] master.ServerManager(517): Registering regionserver=fdc0a393186c,46083,1732161297087 2024-11-21T03:54:58,454 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42431 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fdc0a393186c,37035,1732161296933 2024-11-21T03:54:58,454 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42431 {}] master.ServerManager(517): Registering regionserver=fdc0a393186c,37035,1732161296933 2024-11-21T03:54:58,458 DEBUG [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 2024-11-21T03:54:58,458 DEBUG [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 2024-11-21T03:54:58,458 DEBUG [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T03:54:58,458 DEBUG [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T03:54:58,459 DEBUG [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T03:54:58,459 DEBUG [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T03:54:58,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T03:54:58,527 DEBUG [RS:1;fdc0a393186c:46083 {}] zookeeper.ZKUtil(111): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fdc0a393186c,46083,1732161297087 2024-11-21T03:54:58,527 WARN [RS:1;fdc0a393186c:46083 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:54:58,527 DEBUG [RS:0;fdc0a393186c:37035 {}] zookeeper.ZKUtil(111): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fdc0a393186c,37035,1732161296933 2024-11-21T03:54:58,527 INFO [RS:1;fdc0a393186c:46083 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:54:58,528 WARN [RS:0;fdc0a393186c:37035 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:54:58,528 DEBUG [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,46083,1732161297087 2024-11-21T03:54:58,528 INFO [RS:0;fdc0a393186c:37035 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:54:58,528 DEBUG [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,37035,1732161296933 2024-11-21T03:54:58,529 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fdc0a393186c,46083,1732161297087] 2024-11-21T03:54:58,529 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fdc0a393186c,37035,1732161296933] 2024-11-21T03:54:58,553 WARN [fdc0a393186c:42431 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T03:54:58,556 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T03:54:58,556 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T03:54:58,576 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T03:54:58,576 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T03:54:58,580 INFO [RS:0;fdc0a393186c:37035 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T03:54:58,580 INFO [RS:1;fdc0a393186c:46083 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T03:54:58,581 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,581 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,582 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T03:54:58,582 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T03:54:58,587 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T03:54:58,587 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T03:54:58,588 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,588 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,589 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,589 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,589 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,589 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,589 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,589 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,589 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,589 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,589 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,590 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fdc0a393186c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:54:58,590 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,590 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,590 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fdc0a393186c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:54:58,590 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,590 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,590 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,590 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,590 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,590 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,590 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,590 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,591 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:54:58,591 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,591 DEBUG [RS:0;fdc0a393186c:37035 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:54:58,591 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,591 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:54:58,591 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:54:58,591 DEBUG [RS:1;fdc0a393186c:46083 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:54:58,592 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,592 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,592 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,592 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,592 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,592 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,592 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,593 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,593 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,37035,1732161296933-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:54:58,593 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,593 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,593 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,46083,1732161297087-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:54:58,613 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T03:54:58,614 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T03:54:58,615 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,46083,1732161297087-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,615 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,37035,1732161296933-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,615 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,615 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,615 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.Replication(171): fdc0a393186c,46083,1732161297087 started 2024-11-21T03:54:58,615 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.Replication(171): fdc0a393186c,37035,1732161296933 started 2024-11-21T03:54:58,634 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,634 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(1482): Serving as fdc0a393186c,46083,1732161297087, RpcServer on fdc0a393186c/172.17.0.2:46083, sessionid=0x1015cd1686c0002 2024-11-21T03:54:58,635 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T03:54:58,635 DEBUG [RS:1;fdc0a393186c:46083 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fdc0a393186c,46083,1732161297087 2024-11-21T03:54:58,636 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,46083,1732161297087' 2024-11-21T03:54:58,636 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T03:54:58,637 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T03:54:58,638 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T03:54:58,638 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T03:54:58,638 DEBUG [RS:1;fdc0a393186c:46083 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fdc0a393186c,46083,1732161297087 2024-11-21T03:54:58,638 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,46083,1732161297087' 2024-11-21T03:54:58,638 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T03:54:58,639 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T03:54:58,639 DEBUG [RS:1;fdc0a393186c:46083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T03:54:58,639 INFO [RS:1;fdc0a393186c:46083 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T03:54:58,640 INFO [RS:1;fdc0a393186c:46083 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T03:54:58,640 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:58,640 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(1482): Serving as fdc0a393186c,37035,1732161296933, RpcServer on fdc0a393186c/172.17.0.2:37035, sessionid=0x1015cd1686c0001 2024-11-21T03:54:58,640 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T03:54:58,640 DEBUG [RS:0;fdc0a393186c:37035 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fdc0a393186c,37035,1732161296933 2024-11-21T03:54:58,641 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,37035,1732161296933' 2024-11-21T03:54:58,641 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T03:54:58,641 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T03:54:58,642 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T03:54:58,642 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T03:54:58,642 DEBUG [RS:0;fdc0a393186c:37035 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fdc0a393186c,37035,1732161296933 2024-11-21T03:54:58,642 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,37035,1732161296933' 2024-11-21T03:54:58,642 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T03:54:58,643 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T03:54:58,644 DEBUG [RS:0;fdc0a393186c:37035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T03:54:58,644 INFO [RS:0;fdc0a393186c:37035 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T03:54:58,644 INFO [RS:0;fdc0a393186c:37035 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T03:54:58,744 INFO [RS:1;fdc0a393186c:46083 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:54:58,745 INFO [RS:0;fdc0a393186c:37035 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:54:58,745 INFO [RS:1;fdc0a393186c:46083 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fdc0a393186c%2C46083%2C1732161297087, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,46083,1732161297087, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs, maxLogs=55 2024-11-21T03:54:58,746 INFO [RS:0;fdc0a393186c:37035 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fdc0a393186c%2C37035%2C1732161296933, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,37035,1732161296933, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs, maxLogs=55 2024-11-21T03:54:58,765 INFO [RS:1;fdc0a393186c:46083 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,46083,1732161297087/fdc0a393186c%2C46083%2C1732161297087.1732161298748 2024-11-21T03:54:58,765 INFO [RS:0;fdc0a393186c:37035 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,37035,1732161296933/fdc0a393186c%2C37035%2C1732161296933.1732161298748 2024-11-21T03:54:58,767 DEBUG [RS:0;fdc0a393186c:37035 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T03:54:58,767 DEBUG [RS:1;fdc0a393186c:46083 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T03:54:58,806 DEBUG [fdc0a393186c:42431 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T03:54:58,817 DEBUG [fdc0a393186c:42431 {}] balancer.BalancerClusterState(204): Hosts are {fdc0a393186c=0} racks are {/default-rack=0} 2024-11-21T03:54:58,820 DEBUG [fdc0a393186c:42431 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T03:54:58,820 DEBUG [fdc0a393186c:42431 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T03:54:58,821 DEBUG [fdc0a393186c:42431 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T03:54:58,821 DEBUG [fdc0a393186c:42431 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T03:54:58,821 INFO [fdc0a393186c:42431 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T03:54:58,821 INFO [fdc0a393186c:42431 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T03:54:58,821 DEBUG [fdc0a393186c:42431 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T03:54:58,827 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fdc0a393186c,46083,1732161297087 2024-11-21T03:54:58,831 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fdc0a393186c,46083,1732161297087, state=OPENING 2024-11-21T03:54:58,868 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T03:54:58,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:54:58,881 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:54:58,881 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:54:58,881 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:54:58,885 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T03:54:58,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fdc0a393186c,46083,1732161297087}] 2024-11-21T03:54:59,063 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T03:54:59,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55549, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T03:54:59,074 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T03:54:59,074 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:54:59,075 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T03:54:59,076 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fdc0a393186c%2C46083%2C1732161297087.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,46083,1732161297087, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs, maxLogs=55 2024-11-21T03:54:59,094 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,46083,1732161297087/fdc0a393186c%2C46083%2C1732161297087.meta.1732161299078.meta 2024-11-21T03:54:59,094 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T03:54:59,095 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:54:59,097 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T03:54:59,099 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T03:54:59,103 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T03:54:59,107 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T03:54:59,107 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:54:59,107 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T03:54:59,108 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T03:54:59,109 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T03:54:59,110 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T03:54:59,110 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:59,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:54:59,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T03:54:59,112 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T03:54:59,112 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:59,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:54:59,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T03:54:59,113 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T03:54:59,113 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:59,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:54:59,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T03:54:59,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T03:54:59,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:59,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:54:59,115 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T03:54:59,116 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740 2024-11-21T03:54:59,116 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740 2024-11-21T03:54:59,117 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T03:54:59,117 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T03:54:59,117 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T03:54:59,118 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T03:54:59,118 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61652137, jitterRate=-0.08131156861782074}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T03:54:59,119 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T03:54:59,120 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732161299108Writing region info on filesystem at 1732161299109 (+1 ms)Initializing all the Stores at 1732161299109Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161299109Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161299109Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161299109Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161299109Cleaning up temporary data from old regions at 1732161299117 (+8 ms)Running coprocessor post-open hooks at 1732161299119 (+2 ms)Region opened successfully at 1732161299120 (+1 ms) 2024-11-21T03:54:59,125 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732161299053 2024-11-21T03:54:59,135 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T03:54:59,136 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T03:54:59,136 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fdc0a393186c,46083,1732161297087 2024-11-21T03:54:59,138 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fdc0a393186c,46083,1732161297087, state=OPEN 2024-11-21T03:54:59,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:54:59,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:54:59,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:54:59,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:54:59,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:54:59,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:54:59,172 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fdc0a393186c,46083,1732161297087 2024-11-21T03:54:59,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T03:54:59,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fdc0a393186c,46083,1732161297087 in 283 msec 2024-11-21T03:54:59,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T03:54:59,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 789 msec 2024-11-21T03:54:59,185 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:54:59,185 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T03:54:59,206 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:54:59,207 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,46083,1732161297087, seqNum=-1] 2024-11-21T03:54:59,226 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:54:59,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53077, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:54:59,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0280 sec 2024-11-21T03:54:59,244 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732161299244, completionTime=-1 2024-11-21T03:54:59,247 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-21T03:54:59,247 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T03:54:59,270 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T03:54:59,270 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732161359270 2024-11-21T03:54:59,271 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732161419270 2024-11-21T03:54:59,271 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-21T03:54:59,273 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T03:54:59,278 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,42431,1732161296085-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:59,279 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,42431,1732161296085-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:59,279 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,42431,1732161296085-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:59,280 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fdc0a393186c:42431, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:59,280 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:59,281 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:59,287 DEBUG [master/fdc0a393186c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T03:54:59,309 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.055sec 2024-11-21T03:54:59,311 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T03:54:59,312 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T03:54:59,313 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T03:54:59,314 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T03:54:59,314 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T03:54:59,314 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,42431,1732161296085-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:54:59,315 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,42431,1732161296085-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T03:54:59,316 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T03:54:59,317 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T03:54:59,317 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,42431,1732161296085-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:54:59,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0b20dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:54:59,373 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T03:54:59,373 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T03:54:59,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,42431,-1 for getting cluster id 2024-11-21T03:54:59,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:54:59,387 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4d3b120e-c749-4c90-8e40-a29ac6eae3be' 2024-11-21T03:54:59,390 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:54:59,391 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4d3b120e-c749-4c90-8e40-a29ac6eae3be" 2024-11-21T03:54:59,391 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe00d9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:54:59,391 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,42431,-1] 2024-11-21T03:54:59,395 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:54:59,397 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:54:59,398 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:54:59,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3302ae21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:54:59,404 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:54:59,411 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,46083,1732161297087, seqNum=-1] 2024-11-21T03:54:59,412 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:54:59,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:54:59,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fdc0a393186c,42431,1732161296085 2024-11-21T03:54:59,449 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:54:59,454 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fdc0a393186c,42431,1732161296085 2024-11-21T03:54:59,456 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5138e345 2024-11-21T03:54:59,460 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:54:59,463 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:54:59,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732161299445', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:54:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732161299445 2024-11-21T03:54:59,478 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732161299445 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:54:59,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732161299445" procId is: 4 2024-11-21T03:54:59,482 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:59,484 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732161299445 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:54:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:54:59,497 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732161299445-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7451c951cad533b7ca22bd19ce060ffb, NAME => 'testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732161299445', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 2024-11-21T03:54:59,503 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732161299445-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:54:59,503 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732161299445-pool-0 {}] regionserver.HRegion(1722): Closing 7451c951cad533b7ca22bd19ce060ffb, disabling compactions & flushes 2024-11-21T03:54:59,503 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732161299445-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:54:59,503 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732161299445-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:54:59,503 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732161299445-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. after waiting 0 ms 2024-11-21T03:54:59,503 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732161299445-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:54:59,504 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732161299445-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:54:59,504 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732161299445-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7451c951cad533b7ca22bd19ce060ffb: Waiting for close lock at 1732161299503Disabling compacts and flushes for region at 1732161299503Disabling writes for close at 1732161299503Writing region close event to WAL at 1732161299503Closed at 1732161299503 2024-11-21T03:54:59,505 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732161299445 execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:54:59,509 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732161299505"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732161299505"}]},"ts":"1732161299505"} 2024-11-21T03:54:59,513 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:54:59,514 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732161299445 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:54:59,516 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732161299445","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732161299514"}]},"ts":"1732161299514"} 2024-11-21T03:54:59,520 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732161299445, state=ENABLING in hbase:meta 2024-11-21T03:54:59,520 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {fdc0a393186c=0} racks are {/default-rack=0} 2024-11-21T03:54:59,521 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T03:54:59,521 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T03:54:59,521 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T03:54:59,521 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T03:54:59,521 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T03:54:59,521 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T03:54:59,521 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T03:54:59,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, ASSIGN}] 2024-11-21T03:54:59,524 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, ASSIGN 2024-11-21T03:54:59,525 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, ASSIGN; state=OFFLINE, location=fdc0a393186c,46083,1732161297087; forceNewPlan=false, retain=false 2024-11-21T03:54:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:54:59,678 INFO [fdc0a393186c:42431 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T03:54:59,680 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7451c951cad533b7ca22bd19ce060ffb, regionState=OPENING, regionLocation=fdc0a393186c,46083,1732161297087 2024-11-21T03:54:59,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, ASSIGN because future has completed 2024-11-21T03:54:59,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7451c951cad533b7ca22bd19ce060ffb, server=fdc0a393186c,46083,1732161297087}] 2024-11-21T03:54:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:54:59,847 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:54:59,847 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7451c951cad533b7ca22bd19ce060ffb, NAME => 'testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:54:59,847 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732161299445 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,848 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:54:59,848 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,848 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,849 INFO [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,852 INFO [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7451c951cad533b7ca22bd19ce060ffb columnFamilyName family 2024-11-21T03:54:59,853 DEBUG [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:54:59,853 INFO [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] regionserver.HStore(327): Store=7451c951cad533b7ca22bd19ce060ffb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:54:59,853 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,854 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,855 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,855 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,855 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,856 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,861 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:54:59,862 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7451c951cad533b7ca22bd19ce060ffb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66404853, jitterRate=-0.010490581393241882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T03:54:59,862 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:54:59,863 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7451c951cad533b7ca22bd19ce060ffb: Running coprocessor pre-open hook at 1732161299848Writing region info on filesystem at 1732161299848Initializing all the Stores at 1732161299848Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161299848Cleaning up temporary data from old regions at 1732161299855 (+7 ms)Running coprocessor post-open hooks at 1732161299862 (+7 ms)Region opened successfully at 1732161299863 (+1 ms) 2024-11-21T03:54:59,864 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb., pid=6, masterSystemTime=1732161299841 2024-11-21T03:54:59,867 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:54:59,867 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:54:59,868 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7451c951cad533b7ca22bd19ce060ffb, regionState=OPEN, openSeqNum=2, regionLocation=fdc0a393186c,46083,1732161297087 2024-11-21T03:54:59,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7451c951cad533b7ca22bd19ce060ffb, server=fdc0a393186c,46083,1732161297087 because future has completed 2024-11-21T03:54:59,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T03:54:59,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7451c951cad533b7ca22bd19ce060ffb, server=fdc0a393186c,46083,1732161297087 in 186 msec 2024-11-21T03:54:59,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T03:54:59,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, ASSIGN in 353 msec 2024-11-21T03:54:59,879 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732161299445 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:54:59,879 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732161299445","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732161299879"}]},"ts":"1732161299879"} 2024-11-21T03:54:59,882 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732161299445, state=ENABLED in hbase:meta 2024-11-21T03:54:59,883 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732161299445 execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:54:59,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732161299445 in 412 msec 2024-11-21T03:55:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:55:00,120 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732161299445 completed 2024-11-21T03:55:00,120 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732161299445 get assigned. Timeout = 60000ms 2024-11-21T03:55:00,123 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T03:55:00,131 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732161299445 assigned to meta. Checking AM states. 2024-11-21T03:55:00,131 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T03:55:00,132 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732161299445 assigned. 2024-11-21T03:55:00,139 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732161299445', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb., hostname=fdc0a393186c,46083,1732161297087, seqNum=2] 2024-11-21T03:55:00,151 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7451c951cad533b7ca22bd19ce060ffb 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-21T03:55:00,175 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 95162dbd6c554e61891099f0e642b940 2024-11-21T03:55:00,213 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/.tmp/family/95162dbd6c554e61891099f0e642b940 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-21T03:55:00,218 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/.tmp/family/95162dbd6c554e61891099f0e642b940 2024-11-21T03:55:00,248 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/.tmp/family/95162dbd6c554e61891099f0e642b940 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940 2024-11-21T03:55:00,250 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940, delay=965 ms 2024-11-21T03:55:00,252 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940, entries=2, sequenceid=6, filesize=4.9 K 2024-11-21T03:55:00,258 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 7451c951cad533b7ca22bd19ce060ffb in 107ms, sequenceid=6, compaction requested=false 2024-11-21T03:55:00,259 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732161299445' 2024-11-21T03:55:00,261 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7451c951cad533b7ca22bd19ce060ffb: 2024-11-21T03:55:01,217 DEBUG [hfile-prefetch-1732161300250 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 95162dbd6c554e61891099f0e642b940_0. 2024-11-21T03:55:01,221 DEBUG [hfile-prefetch-1732161300250 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 95162dbd6c554e61891099f0e642b940_117. 2024-11-21T03:55:01,222 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940, with total blocks 2, and data blocks 1 2024-11-21T03:55:01,222 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-21T03:55:01,222 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-21T03:55:01,222 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940. Will sleep 100ms and try the verification again. 2024-11-21T03:55:01,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=7451c951cad533b7ca22bd19ce060ffb, source=fdc0a393186c,46083,1732161297087, destination=fdc0a393186c,37035,1732161296933, warming up region on fdc0a393186c,37035,1732161296933 2024-11-21T03:55:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T03:55:01,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=7451c951cad533b7ca22bd19ce060ffb, source=fdc0a393186c,46083,1732161297087, destination=fdc0a393186c,37035,1732161296933, running balancer 2024-11-21T03:55:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, REOPEN/MOVE 2024-11-21T03:55:01,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57047, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T03:55:01,276 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, REOPEN/MOVE 2024-11-21T03:55:01,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37035 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37035 {}] regionserver.HRegion(7855): Warmup {ENCODED => 7451c951cad533b7ca22bd19ce060ffb, NAME => 'testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:55:01,278 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7451c951cad533b7ca22bd19ce060ffb, regionState=CLOSING, regionLocation=fdc0a393186c,46083,1732161297087 2024-11-21T03:55:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37035 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:55:01,279 INFO [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,279 INFO [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7451c951cad533b7ca22bd19ce060ffb columnFamilyName family 2024-11-21T03:55:01,279 DEBUG [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:01,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, REOPEN/MOVE because future has completed 2024-11-21T03:55:01,281 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-21T03:55:01,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7451c951cad533b7ca22bd19ce060ffb, server=fdc0a393186c,46083,1732161297087}] 2024-11-21T03:55:01,283 DEBUG [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940 2024-11-21T03:55:01,283 INFO [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] regionserver.HStore(327): Store=7451c951cad533b7ca22bd19ce060ffb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:55:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37035 {}] regionserver.HRegion(1722): Closing 7451c951cad533b7ca22bd19ce060ffb, disabling compactions & flushes 2024-11-21T03:55:01,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37035 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37035 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37035 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. after waiting 0 ms 2024-11-21T03:55:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37035 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,285 DEBUG [StoreFileCloser-7451c951cad533b7ca22bd19ce060ffb-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940 2024-11-21T03:55:01,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37035 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37035 {}] regionserver.HRegion(1676): Region close journal for 7451c951cad533b7ca22bd19ce060ffb: Waiting for close lock at 1732161301284Disabling compacts and flushes for region at 1732161301284Disabling writes for close at 1732161301284Writing region close event to WAL at 1732161301286 (+2 ms)Closed at 1732161301286 2024-11-21T03:55:01,287 WARN [hfile-prefetch-1732161300250 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-21T03:55:01,288 DEBUG [hfile-prefetch-1732161300250 {}] hfile.PrefetchExecutor(121): Prefetch completed for 95162dbd6c554e61891099f0e642b940 2024-11-21T03:55:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42431 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-21T03:55:01,336 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T03:55:01,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T03:55:01,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T03:55:01,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T03:55:01,337 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T03:55:01,337 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T03:55:01,338 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T03:55:01,338 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T03:55:01,351 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T03:55:01,351 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732161301337 2024-11-21T03:55:01,351 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T03:55:01,354 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-21T03:55:01,354 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732161301336 2024-11-21T03:55:01,354 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T03:55:01,439 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,440 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-21T03:55:01,440 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 7451c951cad533b7ca22bd19ce060ffb, disabling compactions & flushes 2024-11-21T03:55:01,440 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,440 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,441 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. after waiting 0 ms 2024-11-21T03:55:01,441 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,442 DEBUG [StoreFileCloser-7451c951cad533b7ca22bd19ce060ffb-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 95162dbd6c554e61891099f0e642b940, starting offset: 0, end offset: 9223372036854775807 2024-11-21T03:55:01,443 DEBUG [StoreFileCloser-7451c951cad533b7ca22bd19ce060ffb-family-1 {}] bucket.BucketCache(856): removed key 95162dbd6c554e61891099f0e642b940_0 from back map with offset lock 2094080 in the evict process 2024-11-21T03:55:01,447 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-21T03:55:01,448 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,448 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 7451c951cad533b7ca22bd19ce060ffb: Waiting for close lock at 1732161301440Running coprocessor pre-close hooks at 1732161301440Disabling compacts and flushes for region at 1732161301440Disabling writes for close at 1732161301441 (+1 ms)Writing region close event to WAL at 1732161301444 (+3 ms)Running coprocessor post-close hooks at 1732161301448 (+4 ms)Closed at 1732161301448 2024-11-21T03:55:01,448 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 7451c951cad533b7ca22bd19ce060ffb move to fdc0a393186c,37035,1732161296933 record at close sequenceid=6 2024-11-21T03:55:01,451 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T03:55:01,451 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T03:55:01,451 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T03:55:01,451 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T03:55:01,452 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,452 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7451c951cad533b7ca22bd19ce060ffb, regionState=CLOSED 2024-11-21T03:55:01,455 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T03:55:01,455 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732161301451 2024-11-21T03:55:01,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7451c951cad533b7ca22bd19ce060ffb, server=fdc0a393186c,46083,1732161297087 because future has completed 2024-11-21T03:55:01,455 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T03:55:01,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T03:55:01,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 7451c951cad533b7ca22bd19ce060ffb, server=fdc0a393186c,46083,1732161297087 in 174 msec 2024-11-21T03:55:01,460 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, REOPEN/MOVE; state=CLOSED, location=fdc0a393186c,37035,1732161296933; forceNewPlan=false, retain=false 2024-11-21T03:55:01,610 INFO [fdc0a393186c:42431 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T03:55:01,611 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7451c951cad533b7ca22bd19ce060ffb, regionState=OPENING, regionLocation=fdc0a393186c,37035,1732161296933 2024-11-21T03:55:01,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, REOPEN/MOVE because future has completed 2024-11-21T03:55:01,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7451c951cad533b7ca22bd19ce060ffb, server=fdc0a393186c,37035,1732161296933}] 2024-11-21T03:55:01,777 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,777 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 7451c951cad533b7ca22bd19ce060ffb, NAME => 'testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:55:01,778 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732161299445 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,779 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:55:01,779 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,779 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,780 INFO [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,782 INFO [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7451c951cad533b7ca22bd19ce060ffb columnFamilyName family 2024-11-21T03:55:01,782 DEBUG [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:01,785 DEBUG [StoreFileOpener-7451c951cad533b7ca22bd19ce060ffb-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940, delay=910 ms 2024-11-21T03:55:01,787 DEBUG [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940 2024-11-21T03:55:01,787 INFO [StoreOpener-7451c951cad533b7ca22bd19ce060ffb-1 {}] regionserver.HStore(327): Store=7451c951cad533b7ca22bd19ce060ffb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:55:01,787 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,789 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,789 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,790 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,790 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,795 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,796 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 7451c951cad533b7ca22bd19ce060ffb; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73120335, jitterRate=0.08957789838314056}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T03:55:01,796 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,797 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 7451c951cad533b7ca22bd19ce060ffb: Running coprocessor pre-open hook at 1732161301779Writing region info on filesystem at 1732161301779Initializing all the Stores at 1732161301780 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161301780Cleaning up temporary data from old regions at 1732161301790 (+10 ms)Running coprocessor post-open hooks at 1732161301796 (+6 ms)Region opened successfully at 1732161301797 (+1 ms) 2024-11-21T03:55:01,798 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb., pid=9, masterSystemTime=1732161301771 2024-11-21T03:55:01,801 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,801 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,802 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7451c951cad533b7ca22bd19ce060ffb, regionState=OPEN, openSeqNum=10, regionLocation=fdc0a393186c,37035,1732161296933 2024-11-21T03:55:01,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7451c951cad533b7ca22bd19ce060ffb, server=fdc0a393186c,37035,1732161296933 because future has completed 2024-11-21T03:55:01,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-21T03:55:01,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7451c951cad533b7ca22bd19ce060ffb, server=fdc0a393186c,37035,1732161296933 in 190 msec 2024-11-21T03:55:01,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732161299445, region=7451c951cad533b7ca22bd19ce060ffb, REOPEN/MOVE in 535 msec 2024-11-21T03:55:01,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T03:55:01,829 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:55:01,829 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:55:01,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:01,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:01,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:55:01,835 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T03:55:01,835 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-21T03:55:01,836 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fdc0a393186c,42431,1732161296085 2024-11-21T03:55:01,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T03:55:01,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T03:55:01,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T03:55:01,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:01,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:01,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:01,860 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T03:55:01,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:55:01,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:55:01,861 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:55:01,861 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:55:01,861 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:55:01,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:01,862 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fdc0a393186c,37035,1732161296933' ***** 2024-11-21T03:55:01,862 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T03:55:01,862 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fdc0a393186c,46083,1732161297087' ***** 2024-11-21T03:55:01,862 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T03:55:01,862 INFO [RS:0;fdc0a393186c:37035 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T03:55:01,862 INFO [RS:1;fdc0a393186c:46083 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T03:55:01,863 INFO [RS:0;fdc0a393186c:37035 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/bucket.persistence 2024-11-21T03:55:01,863 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T03:55:01,863 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T03:55:01,863 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T03:55:01,863 INFO [RS:1;fdc0a393186c:46083 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/bucket.persistence 2024-11-21T03:55:01,863 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T03:55:01,863 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T03:55:01,863 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T03:55:01,863 INFO [RS:1;fdc0a393186c:46083 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T03:55:01,863 INFO [RS:0;fdc0a393186c:37035 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T03:55:01,863 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T03:55:01,863 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T03:55:01,873 DEBUG [RS:0;fdc0a393186c:37035 {}] bucket.BucketCache(1487): Thread RS:0;fdc0a393186c:37035 started persisting bucket cache to file 2024-11-21T03:55:01,873 DEBUG [RS:1;fdc0a393186c:46083 {}] bucket.BucketCache(1487): Thread RS:1;fdc0a393186c:46083 started persisting bucket cache to file 2024-11-21T03:55:01,874 DEBUG [RS:0;fdc0a393186c:37035 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T03:55:01,874 DEBUG [RS:1;fdc0a393186c:46083 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T03:55:01,874 DEBUG [RS:0;fdc0a393186c:37035 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T03:55:01,874 DEBUG [RS:1;fdc0a393186c:46083 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T03:55:01,877 DEBUG [RS:0;fdc0a393186c:37035 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T03:55:01,877 DEBUG [RS:1;fdc0a393186c:46083 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T03:55:01,877 DEBUG [RS:0;fdc0a393186c:37035 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732161301873 2024-11-21T03:55:01,877 DEBUG [RS:1;fdc0a393186c:46083 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732161301873 2024-11-21T03:55:01,878 DEBUG [RS:0;fdc0a393186c:37035 {}] bucket.BucketCache(1509): Thread RS:0;fdc0a393186c:37035 finished persisting bucket cache to file, renaming 2024-11-21T03:55:01,878 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T03:55:01,878 INFO [RS:0;fdc0a393186c:37035 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T03:55:01,878 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T03:55:01,878 INFO [RS:0;fdc0a393186c:37035 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T03:55:01,878 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(3091): Received CLOSE for 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,879 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(959): stopping server fdc0a393186c,37035,1732161296933 2024-11-21T03:55:01,879 INFO [RS:0;fdc0a393186c:37035 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T03:55:01,879 INFO [RS:0;fdc0a393186c:37035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fdc0a393186c:37035. 2024-11-21T03:55:01,879 DEBUG [RS:0;fdc0a393186c:37035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:55:01,879 DEBUG [RS:0;fdc0a393186c:37035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:01,880 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T03:55:01,880 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T03:55:01,880 DEBUG [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(1325): Online Regions={7451c951cad533b7ca22bd19ce060ffb=testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb.} 2024-11-21T03:55:01,880 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7451c951cad533b7ca22bd19ce060ffb, disabling compactions & flushes 2024-11-21T03:55:01,880 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,880 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,880 DEBUG [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(1351): Waiting on 7451c951cad533b7ca22bd19ce060ffb 2024-11-21T03:55:01,880 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. after waiting 0 ms 2024-11-21T03:55:01,880 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,881 DEBUG [StoreFileCloser-7451c951cad533b7ca22bd19ce060ffb-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/family/95162dbd6c554e61891099f0e642b940 2024-11-21T03:55:01,885 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnRegionMove1732161299445/7451c951cad533b7ca22bd19ce060ffb/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-21T03:55:01,886 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,886 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7451c951cad533b7ca22bd19ce060ffb: Waiting for close lock at 1732161301879Running coprocessor pre-close hooks at 1732161301880 (+1 ms)Disabling compacts and flushes for region at 1732161301880Disabling writes for close at 1732161301880Writing region close event to WAL at 1732161301881 (+1 ms)Running coprocessor post-close hooks at 1732161301886 (+5 ms)Closed at 1732161301886 2024-11-21T03:55:01,886 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb. 2024-11-21T03:55:01,896 DEBUG [RS:1;fdc0a393186c:46083 {}] bucket.BucketCache(1509): Thread RS:1;fdc0a393186c:46083 finished persisting bucket cache to file, renaming 2024-11-21T03:55:01,896 WARN [RS:1;fdc0a393186c:46083 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-21T03:55:01,896 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T03:55:01,896 INFO [RS:1;fdc0a393186c:46083 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T03:55:01,896 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T03:55:01,896 INFO [RS:1;fdc0a393186c:46083 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T03:55:01,896 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(959): stopping server fdc0a393186c,46083,1732161297087 2024-11-21T03:55:01,896 INFO [RS:1;fdc0a393186c:46083 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T03:55:01,896 INFO [RS:1;fdc0a393186c:46083 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fdc0a393186c:46083. 2024-11-21T03:55:01,897 DEBUG [RS:1;fdc0a393186c:46083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:55:01,897 DEBUG [RS:1;fdc0a393186c:46083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:01,897 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T03:55:01,897 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T03:55:01,897 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T03:55:01,897 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T03:55:01,897 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T03:55:01,897 DEBUG [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T03:55:01,897 DEBUG [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T03:55:01,898 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T03:55:01,898 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T03:55:01,898 INFO [regionserver/fdc0a393186c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:55:01,898 INFO [regionserver/fdc0a393186c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:55:01,898 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T03:55:01,898 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T03:55:01,898 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T03:55:01,898 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T03:55:01,898 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-21T03:55:01,908 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 68011727b3544a138df4b8b814cce682 2024-11-21T03:55:01,931 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/info/68011727b3544a138df4b8b814cce682 is 205, key is testBlockEvictionOnRegionMove1732161299445,,1732161299465.7451c951cad533b7ca22bd19ce060ffb./info:regioninfo/1732161301802/Put/seqid=0 2024-11-21T03:55:01,932 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/info/68011727b3544a138df4b8b814cce682 2024-11-21T03:55:01,941 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0fb5b7bf378f4077a0386d930649bb24 2024-11-21T03:55:01,954 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/ns/0fb5b7bf378f4077a0386d930649bb24 is 43, key is default/ns:d/1732161299232/Put/seqid=0 2024-11-21T03:55:01,955 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/ns/0fb5b7bf378f4077a0386d930649bb24 2024-11-21T03:55:01,965 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 677bb21b31f54a0893e1f457dcf8a9f0 2024-11-21T03:55:01,979 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/table/677bb21b31f54a0893e1f457dcf8a9f0 is 78, key is testBlockEvictionOnRegionMove1732161299445/table:state/1732161299879/Put/seqid=0 2024-11-21T03:55:01,980 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/table/677bb21b31f54a0893e1f457dcf8a9f0 2024-11-21T03:55:01,983 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/info/68011727b3544a138df4b8b814cce682 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/68011727b3544a138df4b8b814cce682 2024-11-21T03:55:01,984 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/68011727b3544a138df4b8b814cce682, delay=925 ms 2024-11-21T03:55:01,985 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/68011727b3544a138df4b8b814cce682, entries=15, sequenceid=15, filesize=7.6 K 2024-11-21T03:55:01,985 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/ns/0fb5b7bf378f4077a0386d930649bb24 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/0fb5b7bf378f4077a0386d930649bb24 2024-11-21T03:55:01,986 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/0fb5b7bf378f4077a0386d930649bb24, delay=923 ms 2024-11-21T03:55:01,988 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/0fb5b7bf378f4077a0386d930649bb24, entries=2, sequenceid=15, filesize=5.0 K 2024-11-21T03:55:01,988 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/table/677bb21b31f54a0893e1f457dcf8a9f0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/677bb21b31f54a0893e1f457dcf8a9f0 2024-11-21T03:55:01,989 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/677bb21b31f54a0893e1f457dcf8a9f0, delay=916 ms 2024-11-21T03:55:01,990 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/677bb21b31f54a0893e1f457dcf8a9f0, entries=2, sequenceid=15, filesize=5.3 K 2024-11-21T03:55:01,991 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false 2024-11-21T03:55:01,991 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T03:55:01,992 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/68011727b3544a138df4b8b814cce682 2024-11-21T03:55:01,993 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/0fb5b7bf378f4077a0386d930649bb24 2024-11-21T03:55:01,994 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/677bb21b31f54a0893e1f457dcf8a9f0 2024-11-21T03:55:01,998 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-21T03:55:01,999 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T03:55:01,999 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T03:55:02,000 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732161301897Running coprocessor pre-close hooks at 1732161301898 (+1 ms)Disabling compacts and flushes for region at 1732161301898Disabling writes for close at 1732161301898Obtaining lock to block concurrent updates at 1732161301898Preparing flush snapshotting stores in 1588230740 at 1732161301898Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732161301899 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732161301899Flushing 1588230740/info: creating writer at 1732161301899Flushing 1588230740/info: appending metadata at 1732161301930 (+31 ms)Flushing 1588230740/info: closing flushed file at 1732161301930Flushing 1588230740/ns: creating writer at 1732161301935 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732161301953 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732161301953Flushing 1588230740/table: creating writer at 1732161301959 (+6 ms)Flushing 1588230740/table: appending metadata at 1732161301978 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732161301978Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@155b8af7: reopening flushed file at 1732161301982 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e8ea814: reopening flushed file at 1732161301985 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fffd15c: reopening flushed file at 1732161301988 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false at 1732161301991 (+3 ms)Writing region close event to WAL at 1732161301994 (+3 ms)Running coprocessor post-close hooks at 1732161301999 (+5 ms)Closed at 1732161301999 2024-11-21T03:55:02,000 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T03:55:02,081 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(976): stopping server fdc0a393186c,37035,1732161296933; all regions closed. 2024-11-21T03:55:02,089 DEBUG [RS:0;fdc0a393186c:37035 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs 2024-11-21T03:55:02,089 INFO [RS:0;fdc0a393186c:37035 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fdc0a393186c%2C37035%2C1732161296933:(num 1732161298748) 2024-11-21T03:55:02,090 DEBUG [RS:0;fdc0a393186c:37035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:02,090 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:55:02,090 INFO [RS:0;fdc0a393186c:37035 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T03:55:02,090 INFO [RS:0;fdc0a393186c:37035 {}] hbase.ChoreService(370): Chore service for: regionserver/fdc0a393186c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T03:55:02,091 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T03:55:02,091 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T03:55:02,091 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T03:55:02,091 INFO [regionserver/fdc0a393186c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T03:55:02,091 INFO [RS:0;fdc0a393186c:37035 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T03:55:02,091 INFO [RS:0;fdc0a393186c:37035 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37035 2024-11-21T03:55:02,098 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(976): stopping server fdc0a393186c,46083,1732161297087; all regions closed. 2024-11-21T03:55:02,100 DEBUG [RS:1;fdc0a393186c:46083 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs 2024-11-21T03:55:02,100 INFO [RS:1;fdc0a393186c:46083 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fdc0a393186c%2C46083%2C1732161297087.meta:.meta(num 1732161299078) 2024-11-21T03:55:02,101 DEBUG [RS:1;fdc0a393186c:46083 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs 2024-11-21T03:55:02,102 INFO [RS:1;fdc0a393186c:46083 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fdc0a393186c%2C46083%2C1732161297087:(num 1732161298748) 2024-11-21T03:55:02,102 DEBUG [RS:1;fdc0a393186c:46083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:02,102 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:55:02,102 INFO [RS:1;fdc0a393186c:46083 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T03:55:02,102 INFO [RS:1;fdc0a393186c:46083 {}] hbase.ChoreService(370): Chore service for: regionserver/fdc0a393186c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T03:55:02,102 INFO [RS:1;fdc0a393186c:46083 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T03:55:02,102 INFO [regionserver/fdc0a393186c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T03:55:02,103 INFO [RS:1;fdc0a393186c:46083 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46083 2024-11-21T03:55:02,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T03:55:02,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fdc0a393186c,37035,1732161296933 2024-11-21T03:55:02,111 INFO [RS:0;fdc0a393186c:37035 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T03:55:02,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fdc0a393186c,46083,1732161297087 2024-11-21T03:55:02,120 INFO [RS:1;fdc0a393186c:46083 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T03:55:02,120 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fdc0a393186c,46083,1732161297087] 2024-11-21T03:55:02,136 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fdc0a393186c,46083,1732161297087 already deleted, retry=false 2024-11-21T03:55:02,136 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fdc0a393186c,46083,1732161297087 expired; onlineServers=1 2024-11-21T03:55:02,137 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fdc0a393186c,37035,1732161296933] 2024-11-21T03:55:02,144 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fdc0a393186c,37035,1732161296933 already deleted, retry=false 2024-11-21T03:55:02,144 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fdc0a393186c,37035,1732161296933 expired; onlineServers=0 2024-11-21T03:55:02,145 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fdc0a393186c,42431,1732161296085' ***** 2024-11-21T03:55:02,145 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T03:55:02,145 INFO [M:0;fdc0a393186c:42431 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T03:55:02,145 INFO [M:0;fdc0a393186c:42431 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T03:55:02,145 DEBUG [M:0;fdc0a393186c:42431 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T03:55:02,145 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T03:55:02,145 DEBUG [M:0;fdc0a393186c:42431 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T03:55:02,145 DEBUG [master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.small.0-1732161298327 {}] cleaner.HFileCleaner(306): Exit Thread[master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.small.0-1732161298327,5,FailOnTimeoutGroup] 2024-11-21T03:55:02,145 DEBUG [master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.large.0-1732161298325 {}] cleaner.HFileCleaner(306): Exit Thread[master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.large.0-1732161298325,5,FailOnTimeoutGroup] 2024-11-21T03:55:02,146 INFO [M:0;fdc0a393186c:42431 {}] hbase.ChoreService(370): Chore service for: master/fdc0a393186c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T03:55:02,146 INFO [M:0;fdc0a393186c:42431 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T03:55:02,146 DEBUG [M:0;fdc0a393186c:42431 {}] master.HMaster(1795): Stopping service threads 2024-11-21T03:55:02,146 INFO [M:0;fdc0a393186c:42431 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T03:55:02,146 INFO [M:0;fdc0a393186c:42431 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T03:55:02,147 INFO [M:0;fdc0a393186c:42431 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T03:55:02,147 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T03:55:02,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T03:55:02,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,153 DEBUG [M:0;fdc0a393186c:42431 {}] zookeeper.ZKUtil(347): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T03:55:02,153 WARN [M:0;fdc0a393186c:42431 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T03:55:02,154 INFO [M:0;fdc0a393186c:42431 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/.lastflushedseqids 2024-11-21T03:55:02,163 INFO [M:0;fdc0a393186c:42431 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T03:55:02,163 INFO [M:0;fdc0a393186c:42431 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T03:55:02,164 DEBUG [M:0;fdc0a393186c:42431 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T03:55:02,164 INFO [M:0;fdc0a393186c:42431 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:02,164 DEBUG [M:0;fdc0a393186c:42431 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:02,164 DEBUG [M:0;fdc0a393186c:42431 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T03:55:02,164 DEBUG [M:0;fdc0a393186c:42431 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:02,164 INFO [M:0;fdc0a393186c:42431 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-21T03:55:02,172 DEBUG [M:0;fdc0a393186c:42431 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 64ac13a21a5b4437832b9a97141b2425 2024-11-21T03:55:02,185 DEBUG [M:0;fdc0a393186c:42431 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/64ac13a21a5b4437832b9a97141b2425 is 82, key is hbase:meta,,1/info:regioninfo/1732161299136/Put/seqid=0 2024-11-21T03:55:02,186 INFO [M:0;fdc0a393186c:42431 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/64ac13a21a5b4437832b9a97141b2425 2024-11-21T03:55:02,193 DEBUG [M:0;fdc0a393186c:42431 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f93db63ac22b4ea2969000024379701e 2024-11-21T03:55:02,211 DEBUG [M:0;fdc0a393186c:42431 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f93db63ac22b4ea2969000024379701e is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732161299885/Put/seqid=0 2024-11-21T03:55:02,211 INFO [M:0;fdc0a393186c:42431 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f93db63ac22b4ea2969000024379701e 2024-11-21T03:55:02,219 DEBUG [M:0;fdc0a393186c:42431 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e60563c7d4bb48df96bb11be5dcf0756 2024-11-21T03:55:02,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:02,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37035-0x1015cd1686c0001, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:02,229 INFO [RS:0;fdc0a393186c:37035 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T03:55:02,229 INFO [RS:0;fdc0a393186c:37035 {}] regionserver.HRegionServer(1031): Exiting; stopping=fdc0a393186c,37035,1732161296933; zookeeper connection closed. 2024-11-21T03:55:02,229 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c4f11b3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c4f11b3 2024-11-21T03:55:02,230 DEBUG [M:0;fdc0a393186c:42431 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e60563c7d4bb48df96bb11be5dcf0756 is 69, key is fdc0a393186c,37035,1732161296933/rs:state/1732161298455/Put/seqid=0 2024-11-21T03:55:02,230 INFO [M:0;fdc0a393186c:42431 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e60563c7d4bb48df96bb11be5dcf0756 2024-11-21T03:55:02,232 DEBUG [M:0;fdc0a393186c:42431 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/64ac13a21a5b4437832b9a97141b2425 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/64ac13a21a5b4437832b9a97141b2425 2024-11-21T03:55:02,234 INFO [M:0;fdc0a393186c:42431 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/64ac13a21a5b4437832b9a97141b2425, entries=8, sequenceid=84, filesize=5.5 K 2024-11-21T03:55:02,235 DEBUG [M:0;fdc0a393186c:42431 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f93db63ac22b4ea2969000024379701e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f93db63ac22b4ea2969000024379701e 2024-11-21T03:55:02,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:02,237 INFO [RS:1;fdc0a393186c:46083 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T03:55:02,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46083-0x1015cd1686c0002, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:02,237 INFO [RS:1;fdc0a393186c:46083 {}] regionserver.HRegionServer(1031): Exiting; stopping=fdc0a393186c,46083,1732161297087; zookeeper connection closed. 2024-11-21T03:55:02,237 INFO [M:0;fdc0a393186c:42431 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f93db63ac22b4ea2969000024379701e, entries=9, sequenceid=84, filesize=6.5 K 2024-11-21T03:55:02,237 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5b783c00 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5b783c00 2024-11-21T03:55:02,237 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-21T03:55:02,238 DEBUG [M:0;fdc0a393186c:42431 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e60563c7d4bb48df96bb11be5dcf0756 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e60563c7d4bb48df96bb11be5dcf0756 2024-11-21T03:55:02,240 INFO [M:0;fdc0a393186c:42431 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e60563c7d4bb48df96bb11be5dcf0756, entries=2, sequenceid=84, filesize=5.1 K 2024-11-21T03:55:02,241 INFO [M:0;fdc0a393186c:42431 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=84, compaction requested=false 2024-11-21T03:55:02,243 INFO [M:0;fdc0a393186c:42431 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:02,243 DEBUG [M:0;fdc0a393186c:42431 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732161302163Disabling compacts and flushes for region at 1732161302163Disabling writes for close at 1732161302164 (+1 ms)Obtaining lock to block concurrent updates at 1732161302164Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732161302164Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732161302164Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732161302165 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732161302165Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732161302184 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732161302185 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732161302187 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732161302210 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732161302210Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732161302214 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732161302229 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732161302229Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ccfc570: reopening flushed file at 1732161302232 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fa73015: reopening flushed file at 1732161302234 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68bb6fc6: reopening flushed file at 1732161302237 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=84, compaction requested=false at 1732161302241 (+4 ms)Writing region close event to WAL at 1732161302243 (+2 ms)Closed at 1732161302243 2024-11-21T03:55:02,245 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T03:55:02,245 INFO [M:0;fdc0a393186c:42431 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T03:55:02,245 INFO [M:0;fdc0a393186c:42431 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42431 2024-11-21T03:55:02,246 INFO [M:0;fdc0a393186c:42431 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T03:55:02,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:02,376 INFO [M:0;fdc0a393186c:42431 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T03:55:02,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42431-0x1015cd1686c0000, quorum=127.0.0.1:62150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:02,384 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T03:55:02,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T03:55:02,406 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:02,414 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fdc0a393186c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1732161300250 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732161301984 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/fdc0a393186c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1732161301785 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/fdc0a393186c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732161301986 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fdc0a393186c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=373 (was 388), ProcessCount=11 (was 11), AvailableMemoryMB=6760 (was 7187) 2024-11-21T03:55:02,422 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=373, ProcessCount=11, AvailableMemoryMB=6760 2024-11-21T03:55:02,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/hadoop-log-dir so I do NOT create it in target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8 2024-11-21T03:55:02,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/hadoop-tmp-dir so I do NOT create it in target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8 2024-11-21T03:55:02,423 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:02,425 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/cluster_03a6ec07-aaf0-2699-82f1-e02b6a559e46, deleteOnExit=true 2024-11-21T03:55:02,429 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/cluster_03a6ec07-aaf0-2699-82f1-e02b6a559e46/zookeeper_0, clientPort=58906, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/cluster_03a6ec07-aaf0-2699-82f1-e02b6a559e46/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/cluster_03a6ec07-aaf0-2699-82f1-e02b6a559e46/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T03:55:02,430 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58906 2024-11-21T03:55:02,440 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 with version=8 2024-11-21T03:55:02,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/hbase-staging 2024-11-21T03:55:02,443 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fdc0a393186c:0 server-side Connection retries=45 2024-11-21T03:55:02,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:02,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:02,443 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:55:02,444 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:02,444 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:55:02,444 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T03:55:02,444 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:55:02,445 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34369 2024-11-21T03:55:02,446 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34369 connecting to ZooKeeper ensemble=127.0.0.1:58906 2024-11-21T03:55:02,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:343690x0, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:55:02,491 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34369-0x1015cd182610000 connected 2024-11-21T03:55:02,574 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:02,575 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:02,577 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:55:02,578 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1, hbase.cluster.distributed=false 2024-11-21T03:55:02,582 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:55:02,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34369 2024-11-21T03:55:02,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34369 2024-11-21T03:55:02,584 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34369 2024-11-21T03:55:02,585 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34369 2024-11-21T03:55:02,585 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34369 2024-11-21T03:55:02,603 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fdc0a393186c:0 server-side Connection retries=45 2024-11-21T03:55:02,603 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:02,603 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:02,604 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:55:02,604 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:02,604 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:55:02,604 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T03:55:02,604 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:55:02,605 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45259 2024-11-21T03:55:02,606 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45259 connecting to ZooKeeper ensemble=127.0.0.1:58906 2024-11-21T03:55:02,607 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:02,607 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:02,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452590x0, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:55:02,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45259-0x1015cd182610001 connected 2024-11-21T03:55:02,616 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:55:02,616 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T03:55:02,617 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.cache 2024-11-21T03:55:02,617 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T03:55:02,617 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T03:55:02,617 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T03:55:02,618 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T03:55:02,618 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T03:55:02,618 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T03:55:02,618 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T03:55:02,619 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T03:55:02,619 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.persistence is complete. 2024-11-21T03:55:02,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T03:55:02,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:55:02,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45259 2024-11-21T03:55:02,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45259 2024-11-21T03:55:02,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45259 2024-11-21T03:55:02,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45259 2024-11-21T03:55:02,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45259 2024-11-21T03:55:02,641 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fdc0a393186c:0 server-side Connection retries=45 2024-11-21T03:55:02,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:02,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:02,641 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:55:02,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:02,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:55:02,642 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T03:55:02,642 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:55:02,643 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44887 2024-11-21T03:55:02,644 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44887 connecting to ZooKeeper ensemble=127.0.0.1:58906 2024-11-21T03:55:02,644 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:02,644 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:02,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448870x0, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:55:02,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44887-0x1015cd182610002 connected 2024-11-21T03:55:02,658 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:55:02,658 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T03:55:02,659 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.cache 2024-11-21T03:55:02,659 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T03:55:02,659 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T03:55:02,659 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T03:55:02,660 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T03:55:02,660 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T03:55:02,660 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T03:55:02,660 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T03:55:02,660 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.persistence is complete. 2024-11-21T03:55:02,660 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T03:55:02,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T03:55:02,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:55:02,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44887 2024-11-21T03:55:02,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44887 2024-11-21T03:55:02,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44887 2024-11-21T03:55:02,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44887 2024-11-21T03:55:02,669 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44887 2024-11-21T03:55:02,680 DEBUG [M:0;fdc0a393186c:34369 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fdc0a393186c:34369 2024-11-21T03:55:02,681 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fdc0a393186c,34369,1732161302442 2024-11-21T03:55:02,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:55:02,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:55:02,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:55:02,691 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fdc0a393186c,34369,1732161302442 2024-11-21T03:55:02,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T03:55:02,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T03:55:02,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,700 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T03:55:02,700 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fdc0a393186c,34369,1732161302442 from backup master directory 2024-11-21T03:55:02,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fdc0a393186c,34369,1732161302442 2024-11-21T03:55:02,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:55:02,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:55:02,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:55:02,707 WARN [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:55:02,707 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fdc0a393186c,34369,1732161302442 2024-11-21T03:55:02,709 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/hbase.id] with ID: 03534b38-04c2-4ebb-8724-f3357f7788bb 2024-11-21T03:55:02,709 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/.tmp/hbase.id 2024-11-21T03:55:02,715 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/hbase.id] 2024-11-21T03:55:02,753 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:02,753 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T03:55:02,755 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-21T03:55:02,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,770 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:55:02,771 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T03:55:02,771 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:55:02,788 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store 2024-11-21T03:55:02,794 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:55:02,794 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T03:55:02,794 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:02,795 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:02,795 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T03:55:02,795 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:02,795 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:02,795 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732161302794Disabling compacts and flushes for region at 1732161302794Disabling writes for close at 1732161302795 (+1 ms)Writing region close event to WAL at 1732161302795Closed at 1732161302795 2024-11-21T03:55:02,795 WARN [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/.initializing 2024-11-21T03:55:02,795 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/WALs/fdc0a393186c,34369,1732161302442 2024-11-21T03:55:02,801 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:55:02,804 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fdc0a393186c%2C34369%2C1732161302442, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/WALs/fdc0a393186c,34369,1732161302442, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/oldWALs, maxLogs=10 2024-11-21T03:55:02,820 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/WALs/fdc0a393186c,34369,1732161302442/fdc0a393186c%2C34369%2C1732161302442.1732161302805 2024-11-21T03:55:02,821 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T03:55:02,821 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:55:02,821 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:55:02,821 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,821 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T03:55:02,827 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:02,827 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:02,827 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T03:55:02,830 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:02,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:55:02,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T03:55:02,834 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:02,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:55:02,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T03:55:02,837 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:02,838 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:55:02,838 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,839 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,839 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,839 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,839 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,840 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T03:55:02,840 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:55:02,846 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:55:02,846 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58796566, jitterRate=-0.12386289238929749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T03:55:02,847 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732161302822Initializing all the Stores at 1732161302822Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161302822Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161302822Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161302822Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161302822Cleaning up temporary data from old regions at 1732161302839 (+17 ms)Region opened successfully at 1732161302847 (+8 ms) 2024-11-21T03:55:02,848 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T03:55:02,851 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322c484d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fdc0a393186c/172.17.0.2:0 2024-11-21T03:55:02,853 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T03:55:02,853 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T03:55:02,853 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T03:55:02,853 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T03:55:02,853 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T03:55:02,854 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-21T03:55:02,854 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T03:55:02,856 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T03:55:02,857 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T03:55:02,881 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T03:55:02,881 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T03:55:02,882 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T03:55:02,890 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T03:55:02,891 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T03:55:02,893 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T03:55:02,899 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T03:55:02,901 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T03:55:02,907 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T03:55:02,912 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T03:55:02,922 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T03:55:02,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T03:55:02,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T03:55:02,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T03:55:02,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,933 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fdc0a393186c,34369,1732161302442, sessionid=0x1015cd182610000, setting cluster-up flag (Was=false) 2024-11-21T03:55:02,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,974 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T03:55:02,977 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fdc0a393186c,34369,1732161302442 2024-11-21T03:55:02,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:02,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:03,015 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T03:55:03,017 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fdc0a393186c,34369,1732161302442 2024-11-21T03:55:03,017 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T03:55:03,019 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T03:55:03,019 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T03:55:03,020 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T03:55:03,020 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fdc0a393186c,34369,1732161302442 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T03:55:03,022 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:55:03,022 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:55:03,022 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:55:03,022 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:55:03,022 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fdc0a393186c:0, corePoolSize=10, maxPoolSize=10 2024-11-21T03:55:03,022 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,022 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fdc0a393186c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:55:03,022 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,023 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732161333023 2024-11-21T03:55:03,023 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T03:55:03,023 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T03:55:03,023 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T03:55:03,023 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T03:55:03,023 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T03:55:03,023 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T03:55:03,024 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:55:03,024 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T03:55:03,025 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:03,025 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,025 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T03:55:03,026 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T03:55:03,026 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T03:55:03,026 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T03:55:03,026 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T03:55:03,026 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T03:55:03,027 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.large.0-1732161303026,5,FailOnTimeoutGroup] 2024-11-21T03:55:03,027 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.small.0-1732161303027,5,FailOnTimeoutGroup] 2024-11-21T03:55:03,027 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,027 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T03:55:03,027 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,027 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,036 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T03:55:03,036 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 2024-11-21T03:55:03,041 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:55:03,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T03:55:03,045 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T03:55:03,045 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:03,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:03,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T03:55:03,048 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T03:55:03,048 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:03,048 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:03,049 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T03:55:03,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T03:55:03,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:03,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:03,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T03:55:03,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T03:55:03,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:03,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:03,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T03:55:03,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740 2024-11-21T03:55:03,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740 2024-11-21T03:55:03,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T03:55:03,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T03:55:03,056 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T03:55:03,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T03:55:03,063 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:55:03,063 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62538894, jitterRate=-0.06809785962104797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T03:55:03,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732161303041Initializing all the Stores at 1732161303042 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161303042Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161303042Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161303042Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161303042Cleaning up temporary data from old regions at 1732161303056 (+14 ms)Region opened successfully at 1732161303064 (+8 ms) 2024-11-21T03:55:03,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T03:55:03,064 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T03:55:03,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T03:55:03,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T03:55:03,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T03:55:03,065 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T03:55:03,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732161303064Disabling compacts and flushes for region at 1732161303064Disabling writes for close at 1732161303065 (+1 ms)Writing region close event to WAL at 1732161303065Closed at 1732161303065 2024-11-21T03:55:03,066 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:55:03,067 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T03:55:03,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T03:55:03,068 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T03:55:03,069 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T03:55:03,073 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(746): ClusterId : 03534b38-04c2-4ebb-8724-f3357f7788bb 2024-11-21T03:55:03,073 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(746): ClusterId : 03534b38-04c2-4ebb-8724-f3357f7788bb 2024-11-21T03:55:03,073 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T03:55:03,073 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T03:55:03,092 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T03:55:03,092 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T03:55:03,092 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T03:55:03,092 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T03:55:03,100 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T03:55:03,100 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T03:55:03,100 DEBUG [RS:0;fdc0a393186c:45259 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9000704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fdc0a393186c/172.17.0.2:0 2024-11-21T03:55:03,101 DEBUG [RS:1;fdc0a393186c:44887 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e872721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fdc0a393186c/172.17.0.2:0 2024-11-21T03:55:03,114 DEBUG [RS:0;fdc0a393186c:45259 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fdc0a393186c:45259 2024-11-21T03:55:03,114 DEBUG [RS:1;fdc0a393186c:44887 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fdc0a393186c:44887 2024-11-21T03:55:03,114 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T03:55:03,114 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T03:55:03,114 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T03:55:03,114 DEBUG [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T03:55:03,114 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T03:55:03,114 DEBUG [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T03:55:03,115 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(2659): reportForDuty to master=fdc0a393186c,34369,1732161302442 with port=45259, startcode=1732161302603 2024-11-21T03:55:03,115 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(2659): reportForDuty to master=fdc0a393186c,34369,1732161302442 with port=44887, startcode=1732161302640 2024-11-21T03:55:03,116 DEBUG [RS:1;fdc0a393186c:44887 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T03:55:03,116 DEBUG [RS:0;fdc0a393186c:45259 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T03:55:03,118 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47057, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T03:55:03,118 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45595, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T03:55:03,119 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34369 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fdc0a393186c,45259,1732161302603 2024-11-21T03:55:03,119 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34369 {}] master.ServerManager(517): Registering regionserver=fdc0a393186c,45259,1732161302603 2024-11-21T03:55:03,120 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34369 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fdc0a393186c,44887,1732161302640 2024-11-21T03:55:03,120 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34369 {}] master.ServerManager(517): Registering regionserver=fdc0a393186c,44887,1732161302640 2024-11-21T03:55:03,120 DEBUG [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 2024-11-21T03:55:03,120 DEBUG [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T03:55:03,121 DEBUG [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T03:55:03,122 DEBUG [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 2024-11-21T03:55:03,122 DEBUG [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T03:55:03,122 DEBUG [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T03:55:03,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T03:55:03,162 DEBUG [RS:0;fdc0a393186c:45259 {}] zookeeper.ZKUtil(111): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fdc0a393186c,45259,1732161302603 2024-11-21T03:55:03,163 WARN [RS:0;fdc0a393186c:45259 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:55:03,163 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fdc0a393186c,44887,1732161302640] 2024-11-21T03:55:03,163 INFO [RS:0;fdc0a393186c:45259 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:55:03,163 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fdc0a393186c,45259,1732161302603] 2024-11-21T03:55:03,163 DEBUG [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,45259,1732161302603 2024-11-21T03:55:03,163 DEBUG [RS:1;fdc0a393186c:44887 {}] zookeeper.ZKUtil(111): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fdc0a393186c,44887,1732161302640 2024-11-21T03:55:03,164 WARN [RS:1;fdc0a393186c:44887 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:55:03,164 INFO [RS:1;fdc0a393186c:44887 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:55:03,164 DEBUG [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,44887,1732161302640 2024-11-21T03:55:03,173 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T03:55:03,175 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T03:55:03,175 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T03:55:03,176 INFO [RS:0;fdc0a393186c:45259 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T03:55:03,176 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,176 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T03:55:03,177 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T03:55:03,177 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fdc0a393186c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,177 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,178 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,178 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:55:03,178 DEBUG [RS:0;fdc0a393186c:45259 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:55:03,178 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T03:55:03,178 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,178 INFO [RS:1;fdc0a393186c:44887 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T03:55:03,178 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,178 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,179 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,179 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T03:55:03,179 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,179 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,179 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,45259,1732161302603-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:55:03,179 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T03:55:03,179 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,179 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,179 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,180 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,180 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,180 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,180 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fdc0a393186c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:55:03,180 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,180 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,180 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,180 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,181 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,181 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:03,181 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:55:03,181 DEBUG [RS:1;fdc0a393186c:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:55:03,183 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,183 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,183 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,183 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,183 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,183 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,44887,1732161302640-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:55:03,193 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T03:55:03,193 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,45259,1732161302603-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,193 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,193 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.Replication(171): fdc0a393186c,45259,1732161302603 started 2024-11-21T03:55:03,197 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T03:55:03,197 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,44887,1732161302640-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,197 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,197 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.Replication(171): fdc0a393186c,44887,1732161302640 started 2024-11-21T03:55:03,208 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,208 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(1482): Serving as fdc0a393186c,45259,1732161302603, RpcServer on fdc0a393186c/172.17.0.2:45259, sessionid=0x1015cd182610001 2024-11-21T03:55:03,208 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T03:55:03,208 DEBUG [RS:0;fdc0a393186c:45259 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fdc0a393186c,45259,1732161302603 2024-11-21T03:55:03,208 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,45259,1732161302603' 2024-11-21T03:55:03,208 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T03:55:03,209 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T03:55:03,209 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T03:55:03,209 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T03:55:03,210 DEBUG [RS:0;fdc0a393186c:45259 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fdc0a393186c,45259,1732161302603 2024-11-21T03:55:03,210 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,45259,1732161302603' 2024-11-21T03:55:03,210 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T03:55:03,210 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T03:55:03,211 DEBUG [RS:0;fdc0a393186c:45259 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T03:55:03,211 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,211 INFO [RS:0;fdc0a393186c:45259 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T03:55:03,211 INFO [RS:0;fdc0a393186c:45259 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T03:55:03,211 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(1482): Serving as fdc0a393186c,44887,1732161302640, RpcServer on fdc0a393186c/172.17.0.2:44887, sessionid=0x1015cd182610002 2024-11-21T03:55:03,211 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T03:55:03,211 DEBUG [RS:1;fdc0a393186c:44887 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fdc0a393186c,44887,1732161302640 2024-11-21T03:55:03,211 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,44887,1732161302640' 2024-11-21T03:55:03,211 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T03:55:03,212 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T03:55:03,212 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T03:55:03,212 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T03:55:03,212 DEBUG [RS:1;fdc0a393186c:44887 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fdc0a393186c,44887,1732161302640 2024-11-21T03:55:03,212 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,44887,1732161302640' 2024-11-21T03:55:03,212 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T03:55:03,213 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T03:55:03,213 DEBUG [RS:1;fdc0a393186c:44887 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T03:55:03,214 INFO [RS:1;fdc0a393186c:44887 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T03:55:03,214 INFO [RS:1;fdc0a393186c:44887 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T03:55:03,219 WARN [fdc0a393186c:34369 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T03:55:03,312 INFO [RS:0;fdc0a393186c:45259 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:55:03,314 INFO [RS:0;fdc0a393186c:45259 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fdc0a393186c%2C45259%2C1732161302603, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,45259,1732161302603, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs, maxLogs=55 2024-11-21T03:55:03,315 INFO [RS:1;fdc0a393186c:44887 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:55:03,316 INFO [RS:1;fdc0a393186c:44887 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fdc0a393186c%2C44887%2C1732161302640, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,44887,1732161302640, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs, maxLogs=55 2024-11-21T03:55:03,335 INFO [RS:0;fdc0a393186c:45259 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,45259,1732161302603/fdc0a393186c%2C45259%2C1732161302603.1732161303319 2024-11-21T03:55:03,335 INFO [RS:1;fdc0a393186c:44887 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,44887,1732161302640/fdc0a393186c%2C44887%2C1732161302640.1732161303319 2024-11-21T03:55:03,335 DEBUG [RS:1;fdc0a393186c:44887 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T03:55:03,335 DEBUG [RS:0;fdc0a393186c:45259 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T03:55:03,470 DEBUG [fdc0a393186c:34369 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T03:55:03,470 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(204): Hosts are {fdc0a393186c=0} racks are {/default-rack=0} 2024-11-21T03:55:03,472 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T03:55:03,472 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T03:55:03,472 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T03:55:03,472 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T03:55:03,472 INFO [fdc0a393186c:34369 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T03:55:03,472 INFO [fdc0a393186c:34369 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T03:55:03,472 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T03:55:03,473 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fdc0a393186c,44887,1732161302640 2024-11-21T03:55:03,475 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fdc0a393186c,44887,1732161302640, state=OPENING 2024-11-21T03:55:03,541 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T03:55:03,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:03,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:03,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:03,551 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T03:55:03,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:03,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:03,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fdc0a393186c,44887,1732161302640}] 2024-11-21T03:55:03,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:03,705 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T03:55:03,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41027, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T03:55:03,715 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T03:55:03,715 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:55:03,716 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T03:55:03,717 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fdc0a393186c%2C44887%2C1732161302640.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,44887,1732161302640, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs, maxLogs=55 2024-11-21T03:55:03,734 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,44887,1732161302640/fdc0a393186c%2C44887%2C1732161302640.meta.1732161303718.meta 2024-11-21T03:55:03,734 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T03:55:03,734 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:55:03,735 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T03:55:03,735 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T03:55:03,735 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T03:55:03,735 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T03:55:03,735 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:55:03,735 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T03:55:03,735 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T03:55:03,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T03:55:03,737 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T03:55:03,737 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:03,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:03,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T03:55:03,738 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T03:55:03,738 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:03,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:03,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T03:55:03,739 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T03:55:03,739 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:03,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:03,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T03:55:03,739 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T03:55:03,739 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:03,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:03,740 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T03:55:03,740 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740 2024-11-21T03:55:03,741 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740 2024-11-21T03:55:03,741 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T03:55:03,741 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T03:55:03,741 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T03:55:03,742 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T03:55:03,742 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74197067, jitterRate=0.10562245547771454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T03:55:03,742 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T03:55:03,743 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732161303736Writing region info on filesystem at 1732161303736Initializing all the Stores at 1732161303736Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161303736Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161303736Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161303736Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161303736Cleaning up temporary data from old regions at 1732161303741 (+5 ms)Running coprocessor post-open hooks at 1732161303742 (+1 ms)Region opened successfully at 1732161303743 (+1 ms) 2024-11-21T03:55:03,743 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732161303704 2024-11-21T03:55:03,746 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T03:55:03,746 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T03:55:03,746 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fdc0a393186c,44887,1732161302640 2024-11-21T03:55:03,747 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fdc0a393186c,44887,1732161302640, state=OPEN 2024-11-21T03:55:03,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:55:03,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:55:03,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:55:03,764 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fdc0a393186c,44887,1732161302640 2024-11-21T03:55:03,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:03,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:03,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:03,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T03:55:03,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fdc0a393186c,44887,1732161302640 in 213 msec 2024-11-21T03:55:03,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T03:55:03,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 700 msec 2024-11-21T03:55:03,771 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:55:03,771 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T03:55:03,775 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:55:03,775 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,44887,1732161302640, seqNum=-1] 2024-11-21T03:55:03,776 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:55:03,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52707, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:55:03,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 764 msec 2024-11-21T03:55:03,783 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732161303783, completionTime=-1 2024-11-21T03:55:03,783 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-21T03:55:03,783 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T03:55:03,786 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T03:55:03,786 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732161363786 2024-11-21T03:55:03,786 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732161423786 2024-11-21T03:55:03,786 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-21T03:55:03,786 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T03:55:03,787 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,34369,1732161302442-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,787 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,34369,1732161302442-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,787 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,34369,1732161302442-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,787 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fdc0a393186c:34369, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,787 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,787 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,790 DEBUG [master/fdc0a393186c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T03:55:03,792 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.084sec 2024-11-21T03:55:03,792 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T03:55:03,792 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T03:55:03,792 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T03:55:03,792 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T03:55:03,792 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T03:55:03,792 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,34369,1732161302442-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:55:03,793 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,34369,1732161302442-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T03:55:03,793 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T03:55:03,793 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T03:55:03,793 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,34369,1732161302442-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:03,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:55:03,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,34369,-1 for getting cluster id 2024-11-21T03:55:03,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:55:03,875 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03534b38-04c2-4ebb-8724-f3357f7788bb' 2024-11-21T03:55:03,876 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:55:03,876 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03534b38-04c2-4ebb-8724-f3357f7788bb" 2024-11-21T03:55:03,876 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4765d559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:55:03,876 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,34369,-1] 2024-11-21T03:55:03,877 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:55:03,877 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:03,879 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:55:03,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af0bbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:55:03,881 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:55:03,882 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,44887,1732161302640, seqNum=-1] 2024-11-21T03:55:03,883 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:55:03,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:55:03,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fdc0a393186c,34369,1732161302442 2024-11-21T03:55:03,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:55:03,890 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fdc0a393186c,34369,1732161302442 2024-11-21T03:55:03,890 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@356b1851 2024-11-21T03:55:03,890 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:55:03,892 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:55:03,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34369 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732161303888', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:55:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34369 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732161303888 2024-11-21T03:55:03,896 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732161303888 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:55:03,896 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:03,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34369 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732161303888" procId is: 4 2024-11-21T03:55:03,897 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732161303888 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:55:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34369 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:55:03,906 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732161303888-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6868f7430ca843fd67df9932f7d3b3c9, NAME => 'testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732161303888', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 2024-11-21T03:55:03,913 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732161303888-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:55:03,913 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732161303888-pool-0 {}] regionserver.HRegion(1722): Closing 6868f7430ca843fd67df9932f7d3b3c9, disabling compactions & flushes 2024-11-21T03:55:03,913 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732161303888-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:03,913 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732161303888-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:03,913 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732161303888-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. after waiting 0 ms 2024-11-21T03:55:03,913 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732161303888-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:03,913 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732161303888-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:03,913 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732161303888-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6868f7430ca843fd67df9932f7d3b3c9: Waiting for close lock at 1732161303913Disabling compacts and flushes for region at 1732161303913Disabling writes for close at 1732161303913Writing region close event to WAL at 1732161303913Closed at 1732161303913 2024-11-21T03:55:03,915 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732161303888 execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:55:03,915 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732161303915"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732161303915"}]},"ts":"1732161303915"} 2024-11-21T03:55:03,918 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:55:03,920 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732161303888 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:55:03,920 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732161303888","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732161303920"}]},"ts":"1732161303920"} 2024-11-21T03:55:03,923 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732161303888, state=ENABLING in hbase:meta 2024-11-21T03:55:03,923 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {fdc0a393186c=0} racks are {/default-rack=0} 2024-11-21T03:55:03,924 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T03:55:03,924 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T03:55:03,924 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T03:55:03,924 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T03:55:03,924 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T03:55:03,924 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T03:55:03,924 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T03:55:03,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732161303888, region=6868f7430ca843fd67df9932f7d3b3c9, ASSIGN}] 2024-11-21T03:55:03,926 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732161303888, region=6868f7430ca843fd67df9932f7d3b3c9, ASSIGN 2024-11-21T03:55:03,927 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732161303888, region=6868f7430ca843fd67df9932f7d3b3c9, ASSIGN; state=OFFLINE, location=fdc0a393186c,44887,1732161302640; forceNewPlan=false, retain=false 2024-11-21T03:55:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34369 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:55:04,078 INFO [fdc0a393186c:34369 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T03:55:04,078 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6868f7430ca843fd67df9932f7d3b3c9, regionState=OPENING, regionLocation=fdc0a393186c,44887,1732161302640 2024-11-21T03:55:04,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732161303888, region=6868f7430ca843fd67df9932f7d3b3c9, ASSIGN because future has completed 2024-11-21T03:55:04,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6868f7430ca843fd67df9932f7d3b3c9, server=fdc0a393186c,44887,1732161302640}] 2024-11-21T03:55:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34369 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:55:04,239 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:04,239 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6868f7430ca843fd67df9932f7d3b3c9, NAME => 'testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:55:04,240 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732161303888 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,240 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:55:04,240 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,240 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,240 INFO [StoreOpener-6868f7430ca843fd67df9932f7d3b3c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,243 INFO [StoreOpener-6868f7430ca843fd67df9932f7d3b3c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6868f7430ca843fd67df9932f7d3b3c9 columnFamilyName family 2024-11-21T03:55:04,243 DEBUG [StoreOpener-6868f7430ca843fd67df9932f7d3b3c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:04,244 INFO [StoreOpener-6868f7430ca843fd67df9932f7d3b3c9-1 {}] regionserver.HStore(327): Store=6868f7430ca843fd67df9932f7d3b3c9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:55:04,244 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,244 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,244 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,245 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,245 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,245 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,252 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:55:04,252 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6868f7430ca843fd67df9932f7d3b3c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65559559, jitterRate=-0.023086443543434143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T03:55:04,252 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:04,253 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6868f7430ca843fd67df9932f7d3b3c9: Running coprocessor pre-open hook at 1732161304240Writing region info on filesystem at 1732161304240Initializing all the Stores at 1732161304240Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161304240Cleaning up temporary data from old regions at 1732161304245 (+5 ms)Running coprocessor post-open hooks at 1732161304252 (+7 ms)Region opened successfully at 1732161304253 (+1 ms) 2024-11-21T03:55:04,254 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9., pid=6, masterSystemTime=1732161304237 2024-11-21T03:55:04,257 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:04,257 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:04,257 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6868f7430ca843fd67df9932f7d3b3c9, regionState=OPEN, openSeqNum=2, regionLocation=fdc0a393186c,44887,1732161302640 2024-11-21T03:55:04,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6868f7430ca843fd67df9932f7d3b3c9, server=fdc0a393186c,44887,1732161302640 because future has completed 2024-11-21T03:55:04,263 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T03:55:04,263 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6868f7430ca843fd67df9932f7d3b3c9, server=fdc0a393186c,44887,1732161302640 in 177 msec 2024-11-21T03:55:04,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T03:55:04,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732161303888, region=6868f7430ca843fd67df9932f7d3b3c9, ASSIGN in 339 msec 2024-11-21T03:55:04,265 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732161303888 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:55:04,266 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732161303888","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732161304265"}]},"ts":"1732161304265"} 2024-11-21T03:55:04,268 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732161303888, state=ENABLED in hbase:meta 2024-11-21T03:55:04,269 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732161303888 execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:55:04,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732161303888 in 375 msec 2024-11-21T03:55:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34369 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:55:04,529 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732161303888 completed 2024-11-21T03:55:04,529 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732161303888 get assigned. Timeout = 60000ms 2024-11-21T03:55:04,529 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T03:55:04,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732161303888 assigned to meta. Checking AM states. 2024-11-21T03:55:04,535 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T03:55:04,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732161303888 assigned. 2024-11-21T03:55:04,541 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732161303888', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9., hostname=fdc0a393186c,44887,1732161302640, seqNum=2] 2024-11-21T03:55:04,546 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6868f7430ca843fd67df9932f7d3b3c9 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-21T03:55:04,555 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9bd016c70233446385e73f9d8c29fa64 2024-11-21T03:55:04,567 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/.tmp/family/9bd016c70233446385e73f9d8c29fa64 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-21T03:55:04,567 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/.tmp/family/9bd016c70233446385e73f9d8c29fa64 2024-11-21T03:55:04,570 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/.tmp/family/9bd016c70233446385e73f9d8c29fa64 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64 2024-11-21T03:55:04,571 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64, delay=913 ms 2024-11-21T03:55:04,572 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64, entries=2, sequenceid=6, filesize=4.9 K 2024-11-21T03:55:04,573 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 6868f7430ca843fd67df9932f7d3b3c9 in 27ms, sequenceid=6, compaction requested=false 2024-11-21T03:55:04,573 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732161303888' 2024-11-21T03:55:04,573 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6868f7430ca843fd67df9932f7d3b3c9: 2024-11-21T03:55:04,610 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:55:05,485 DEBUG [hfile-prefetch-1732161300250 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9bd016c70233446385e73f9d8c29fa64_0. 2024-11-21T03:55:05,486 DEBUG [hfile-prefetch-1732161300250 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9bd016c70233446385e73f9d8c29fa64_117. 2024-11-21T03:55:05,486 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64, with total blocks 2, and data blocks 1 2024-11-21T03:55:05,486 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-21T03:55:05,487 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-21T03:55:05,487 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64. Will sleep 100ms and try the verification again. 2024-11-21T03:55:05,562 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T03:55:05,562 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T03:55:05,563 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T03:55:05,563 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T03:55:05,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-21T03:55:05,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732161305562 2024-11-21T03:55:05,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T03:55:05,574 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;fdc0a393186c:44887,5,FailOnTimeoutGroup] 2024-11-21T03:55:05,574 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fdc0a393186c,44887,1732161302640' ***** 2024-11-21T03:55:05,574 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-21T03:55:05,574 INFO [RS:1;fdc0a393186c:44887 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T03:55:05,574 INFO [RS:1;fdc0a393186c:44887 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.persistence 2024-11-21T03:55:05,574 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T03:55:05,574 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T03:55:05,574 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T03:55:05,574 INFO [RS:1;fdc0a393186c:44887 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T03:55:05,574 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T03:55:05,585 DEBUG [RS:1;fdc0a393186c:44887 {}] bucket.BucketCache(1487): Thread RS:1;fdc0a393186c:44887 started persisting bucket cache to file 2024-11-21T03:55:05,585 DEBUG [RS:1;fdc0a393186c:44887 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T03:55:05,585 DEBUG [RS:1;fdc0a393186c:44887 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T03:55:05,586 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T03:55:05,586 INFO [regionserver/fdc0a393186c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:55:05,587 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64, with total blocks 2, and data blocks 1 2024-11-21T03:55:05,587 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-21T03:55:05,587 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2293): found block for file 9bd016c70233446385e73f9d8c29fa64 in the backing map. Acquiring read lock for offset 0 2024-11-21T03:55:05,587 DEBUG [hfile-prefetch-1732161300250 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64 has now been fully cached. 2024-11-21T03:55:05,587 DEBUG [hfile-prefetch-1732161300250 {}] hfile.PrefetchExecutor(121): Prefetch completed for 9bd016c70233446385e73f9d8c29fa64 2024-11-21T03:55:05,588 DEBUG [RS:1;fdc0a393186c:44887 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-21T03:55:05,588 DEBUG [RS:1;fdc0a393186c:44887 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732161305585 2024-11-21T03:55:05,588 DEBUG [RS:1;fdc0a393186c:44887 {}] bucket.BucketCache(1509): Thread RS:1;fdc0a393186c:44887 finished persisting bucket cache to file, renaming 2024-11-21T03:55:05,588 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T03:55:05,588 INFO [RS:1;fdc0a393186c:44887 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T03:55:05,588 INFO [RS:1;fdc0a393186c:44887 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T03:55:05,588 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(3091): Received CLOSE for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:05,588 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(959): stopping server fdc0a393186c,44887,1732161302640 2024-11-21T03:55:05,588 INFO [RS:1;fdc0a393186c:44887 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T03:55:05,588 INFO [RS:1;fdc0a393186c:44887 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fdc0a393186c:44887. 2024-11-21T03:55:05,588 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T03:55:05,589 DEBUG [RS:1;fdc0a393186c:44887 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:55:05,589 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6868f7430ca843fd67df9932f7d3b3c9, disabling compactions & flushes 2024-11-21T03:55:05,589 DEBUG [RS:1;fdc0a393186c:44887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:05,589 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:05,589 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:05,589 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T03:55:05,589 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. after waiting 0 ms 2024-11-21T03:55:05,589 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T03:55:05,589 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:05,589 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T03:55:05,589 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T03:55:05,589 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T03:55:05,589 DEBUG [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(1325): Online Regions={6868f7430ca843fd67df9932f7d3b3c9=testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9., 1588230740=hbase:meta,,1.1588230740} 2024-11-21T03:55:05,589 DEBUG [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:05,589 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T03:55:05,589 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T03:55:05,589 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T03:55:05,589 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T03:55:05,590 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T03:55:05,590 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T03:55:05,590 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-21T03:55:05,594 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-21T03:55:05,594 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:05,594 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6868f7430ca843fd67df9932f7d3b3c9: Waiting for close lock at 1732161305588Running coprocessor pre-close hooks at 1732161305589 (+1 ms)Disabling compacts and flushes for region at 1732161305589Disabling writes for close at 1732161305589Writing region close event to WAL at 1732161305590 (+1 ms)Running coprocessor post-close hooks at 1732161305594 (+4 ms)Closed at 1732161305594 2024-11-21T03:55:05,594 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:05,599 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c26a0da428fb413f94e5ebbc3693a8f8 2024-11-21T03:55:05,610 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/info/c26a0da428fb413f94e5ebbc3693a8f8 is 209, key is testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9./info:regioninfo/1732161304257/Put/seqid=0 2024-11-21T03:55:05,611 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/info/c26a0da428fb413f94e5ebbc3693a8f8 2024-11-21T03:55:05,619 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a1813f5907964b5f8f1a9efc69201442 2024-11-21T03:55:05,631 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/ns/a1813f5907964b5f8f1a9efc69201442 is 43, key is default/ns:d/1732161303778/Put/seqid=0 2024-11-21T03:55:05,632 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/ns/a1813f5907964b5f8f1a9efc69201442 2024-11-21T03:55:05,640 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 80d6aa6760214a35a8935313439fdc6e 2024-11-21T03:55:05,653 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/table/80d6aa6760214a35a8935313439fdc6e is 80, key is testBlockEvictionOnGracefulStop1732161303888/table:state/1732161304265/Put/seqid=0 2024-11-21T03:55:05,653 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/table/80d6aa6760214a35a8935313439fdc6e 2024-11-21T03:55:05,656 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/info/c26a0da428fb413f94e5ebbc3693a8f8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/c26a0da428fb413f94e5ebbc3693a8f8 2024-11-21T03:55:05,658 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/c26a0da428fb413f94e5ebbc3693a8f8, delay=972 ms 2024-11-21T03:55:05,659 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/c26a0da428fb413f94e5ebbc3693a8f8, entries=10, sequenceid=11, filesize=7.0 K 2024-11-21T03:55:05,660 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/ns/a1813f5907964b5f8f1a9efc69201442 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/a1813f5907964b5f8f1a9efc69201442 2024-11-21T03:55:05,661 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/a1813f5907964b5f8f1a9efc69201442, delay=980 ms 2024-11-21T03:55:05,663 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/a1813f5907964b5f8f1a9efc69201442, entries=2, sequenceid=11, filesize=5.0 K 2024-11-21T03:55:05,663 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/table/80d6aa6760214a35a8935313439fdc6e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/80d6aa6760214a35a8935313439fdc6e 2024-11-21T03:55:05,664 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/80d6aa6760214a35a8935313439fdc6e, delay=901 ms 2024-11-21T03:55:05,666 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/80d6aa6760214a35a8935313439fdc6e, entries=2, sequenceid=11, filesize=5.3 K 2024-11-21T03:55:05,667 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=11, compaction requested=false 2024-11-21T03:55:05,667 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/c26a0da428fb413f94e5ebbc3693a8f8 2024-11-21T03:55:05,668 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/a1813f5907964b5f8f1a9efc69201442 2024-11-21T03:55:05,668 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/80d6aa6760214a35a8935313439fdc6e 2024-11-21T03:55:05,672 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-21T03:55:05,672 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T03:55:05,673 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T03:55:05,673 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732161305589Running coprocessor pre-close hooks at 1732161305589Disabling compacts and flushes for region at 1732161305589Disabling writes for close at 1732161305590 (+1 ms)Obtaining lock to block concurrent updates at 1732161305590Preparing flush snapshotting stores in 1588230740 at 1732161305590Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732161305590Flushing stores of hbase:meta,,1.1588230740 at 1732161305590Flushing 1588230740/info: creating writer at 1732161305591 (+1 ms)Flushing 1588230740/info: appending metadata at 1732161305610 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732161305610Flushing 1588230740/ns: creating writer at 1732161305613 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732161305631 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732161305631Flushing 1588230740/table: creating writer at 1732161305634 (+3 ms)Flushing 1588230740/table: appending metadata at 1732161305652 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732161305652Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f771e26: reopening flushed file at 1732161305656 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41b2db39: reopening flushed file at 1732161305660 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@334262d8: reopening flushed file at 1732161305663 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=11, compaction requested=false at 1732161305667 (+4 ms)Writing region close event to WAL at 1732161305669 (+2 ms)Running coprocessor post-close hooks at 1732161305672 (+3 ms)Closed at 1732161305673 (+1 ms) 2024-11-21T03:55:05,673 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T03:55:05,790 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(976): stopping server fdc0a393186c,44887,1732161302640; all regions closed. 2024-11-21T03:55:05,794 DEBUG [RS:1;fdc0a393186c:44887 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs 2024-11-21T03:55:05,795 INFO [RS:1;fdc0a393186c:44887 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fdc0a393186c%2C44887%2C1732161302640.meta:.meta(num 1732161303718) 2024-11-21T03:55:05,799 DEBUG [RS:1;fdc0a393186c:44887 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs 2024-11-21T03:55:05,799 INFO [RS:1;fdc0a393186c:44887 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fdc0a393186c%2C44887%2C1732161302640:(num 1732161303319) 2024-11-21T03:55:05,799 DEBUG [RS:1;fdc0a393186c:44887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:05,799 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:55:05,799 INFO [RS:1;fdc0a393186c:44887 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T03:55:05,800 INFO [RS:1;fdc0a393186c:44887 {}] hbase.ChoreService(370): Chore service for: regionserver/fdc0a393186c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T03:55:05,800 INFO [RS:1;fdc0a393186c:44887 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T03:55:05,801 INFO [regionserver/fdc0a393186c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T03:55:05,801 INFO [RS:1;fdc0a393186c:44887 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44887 2024-11-21T03:55:05,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T03:55:05,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fdc0a393186c,44887,1732161302640 2024-11-21T03:55:05,830 INFO [RS:1;fdc0a393186c:44887 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T03:55:05,840 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fdc0a393186c,44887,1732161302640] 2024-11-21T03:55:05,848 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fdc0a393186c,44887,1732161302640 already deleted, retry=false 2024-11-21T03:55:05,848 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of fdc0a393186c,44887,1732161302640 on fdc0a393186c,34369,1732161302442 2024-11-21T03:55:05,855 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fdc0a393186c,44887,1732161302640, splitWal=true, meta=true 2024-11-21T03:55:05,859 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure fdc0a393186c,44887,1732161302640, splitWal=true, meta=true 2024-11-21T03:55:05,859 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for fdc0a393186c,44887,1732161302640 (carryingMeta=true) fdc0a393186c,44887,1732161302640/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@24e832a6[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-21T03:55:05,860 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fdc0a393186c,44887,1732161302640, splitWal=true, meta=true, isMeta: true 2024-11-21T03:55:05,861 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,44887,1732161302640-splitting 2024-11-21T03:55:05,861 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,44887,1732161302640-splitting dir is empty, no logs to split. 2024-11-21T03:55:05,862 INFO [PEWorker-1 {}] master.SplitWALManager(105): fdc0a393186c,44887,1732161302640 WAL count=0, meta=true 2024-11-21T03:55:05,864 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,44887,1732161302640-splitting dir is empty, no logs to split. 2024-11-21T03:55:05,864 INFO [PEWorker-1 {}] master.SplitWALManager(105): fdc0a393186c,44887,1732161302640 WAL count=0, meta=true 2024-11-21T03:55:05,864 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fdc0a393186c,44887,1732161302640 WAL splitting is done? wals=0, meta=true 2024-11-21T03:55:05,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T03:55:05,867 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T03:55:05,868 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T03:55:05,940 INFO [RS:1;fdc0a393186c:44887 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T03:55:05,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:05,940 INFO [RS:1;fdc0a393186c:44887 {}] regionserver.HRegionServer(1031): Exiting; stopping=fdc0a393186c,44887,1732161302640; zookeeper connection closed. 2024-11-21T03:55:05,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x1015cd182610002, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:05,941 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3cea8da7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3cea8da7 2024-11-21T03:55:06,018 DEBUG [fdc0a393186c:34369 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T03:55:06,019 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fdc0a393186c,45259,1732161302603 2024-11-21T03:55:06,020 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fdc0a393186c,45259,1732161302603, state=OPENING 2024-11-21T03:55:06,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:55:06,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:55:06,040 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T03:55:06,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:06,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:06,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fdc0a393186c,45259,1732161302603}] 2024-11-21T03:55:06,096 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fdc0a393186c:0 server-side Connection retries=45 2024-11-21T03:55:06,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:06,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:06,096 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:55:06,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:55:06,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:55:06,096 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T03:55:06,096 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:55:06,097 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41615 2024-11-21T03:55:06,099 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41615 connecting to ZooKeeper ensemble=127.0.0.1:58906 2024-11-21T03:55:06,099 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:06,099 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:06,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416150x0, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:55:06,123 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-21T03:55:06,123 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41615-0x1015cd182610003 connected 2024-11-21T03:55:06,123 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-21T03:55:06,124 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T03:55:06,125 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.cache 2024-11-21T03:55:06,125 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T03:55:06,125 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T03:55:06,125 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T03:55:06,125 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T03:55:06,126 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T03:55:06,126 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T03:55:06,127 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-21T03:55:06,127 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T03:55:06,129 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-21T03:55:06,130 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:55:06,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41615 2024-11-21T03:55:06,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41615 2024-11-21T03:55:06,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41615 2024-11-21T03:55:06,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41615 2024-11-21T03:55:06,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41615 2024-11-21T03:55:06,133 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-21T03:55:06,134 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T03:55:06,134 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(746): ClusterId : 03534b38-04c2-4ebb-8724-f3357f7788bb 2024-11-21T03:55:06,134 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T03:55:06,134 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-21T03:55:06,134 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.persistence is complete. 2024-11-21T03:55:06,141 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T03:55:06,141 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T03:55:06,149 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T03:55:06,150 DEBUG [RS:2;fdc0a393186c:41615 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@374eaf45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fdc0a393186c/172.17.0.2:0 2024-11-21T03:55:06,165 DEBUG [RS:2;fdc0a393186c:41615 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;fdc0a393186c:41615 2024-11-21T03:55:06,165 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T03:55:06,165 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T03:55:06,165 DEBUG [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T03:55:06,166 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(2659): reportForDuty to master=fdc0a393186c,34369,1732161302442 with port=41615, startcode=1732161306095 2024-11-21T03:55:06,167 DEBUG [RS:2;fdc0a393186c:41615 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T03:55:06,168 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49283, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T03:55:06,169 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34369 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fdc0a393186c,41615,1732161306095 2024-11-21T03:55:06,169 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34369 {}] master.ServerManager(517): Registering regionserver=fdc0a393186c,41615,1732161306095 2024-11-21T03:55:06,170 DEBUG [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1 2024-11-21T03:55:06,170 DEBUG [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T03:55:06,170 DEBUG [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T03:55:06,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T03:55:06,182 DEBUG [RS:2;fdc0a393186c:41615 {}] zookeeper.ZKUtil(111): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fdc0a393186c,41615,1732161306095 2024-11-21T03:55:06,182 WARN [RS:2;fdc0a393186c:41615 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:55:06,182 INFO [RS:2;fdc0a393186c:41615 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:55:06,182 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fdc0a393186c,41615,1732161306095] 2024-11-21T03:55:06,182 DEBUG [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,41615,1732161306095 2024-11-21T03:55:06,188 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T03:55:06,192 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T03:55:06,193 INFO [RS:2;fdc0a393186c:41615 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T03:55:06,193 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T03:55:06,193 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:06,193 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T03:55:06,194 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T03:55:06,194 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:06,195 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,195 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,195 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,195 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32803, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T03:55:06,195 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,195 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fdc0a393186c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:55:06,196 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,196 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,196 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,196 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,196 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,196 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:55:06,197 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:55:06,197 DEBUG [RS:2;fdc0a393186c:41615 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:55:06,198 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:06,198 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T03:55:06,198 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:06,198 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:55:06,198 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:06,198 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:06,198 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:06,198 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,41615,1732161306095-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:55:06,199 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T03:55:06,200 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fdc0a393186c%2C45259%2C1732161302603.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,45259,1732161302603, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs, maxLogs=55 2024-11-21T03:55:06,217 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,45259,1732161302603/fdc0a393186c%2C45259%2C1732161302603.meta.1732161306201.meta 2024-11-21T03:55:06,217 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T03:55:06,217 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:55:06,218 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T03:55:06,218 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T03:55:06,218 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T03:55:06,218 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T03:55:06,218 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:55:06,218 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T03:55:06,218 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T03:55:06,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T03:55:06,220 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T03:55:06,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:06,221 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/c26a0da428fb413f94e5ebbc3693a8f8, delay=987 ms 2024-11-21T03:55:06,221 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T03:55:06,222 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,41615,1732161306095-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:06,222 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:06,222 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.Replication(171): fdc0a393186c,41615,1732161306095 started 2024-11-21T03:55:06,222 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/c26a0da428fb413f94e5ebbc3693a8f8 2024-11-21T03:55:06,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:06,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T03:55:06,223 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T03:55:06,223 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:06,224 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/a1813f5907964b5f8f1a9efc69201442, delay=999 ms 2024-11-21T03:55:06,226 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/a1813f5907964b5f8f1a9efc69201442 2024-11-21T03:55:06,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:06,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T03:55:06,227 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T03:55:06,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:06,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:06,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T03:55:06,227 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T03:55:06,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:06,229 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/80d6aa6760214a35a8935313439fdc6e, delay=925 ms 2024-11-21T03:55:06,230 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/80d6aa6760214a35a8935313439fdc6e 2024-11-21T03:55:06,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:55:06,230 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T03:55:06,231 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740 2024-11-21T03:55:06,231 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740 2024-11-21T03:55:06,232 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T03:55:06,232 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T03:55:06,232 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T03:55:06,233 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T03:55:06,233 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67251420, jitterRate=0.0021242499351501465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T03:55:06,233 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T03:55:06,234 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732161306219Writing region info on filesystem at 1732161306219Initializing all the Stores at 1732161306219Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161306219Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161306219Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161306219Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732161306219Cleaning up temporary data from old regions at 1732161306232 (+13 ms)Running coprocessor post-open hooks at 1732161306233 (+1 ms)Region opened successfully at 1732161306234 (+1 ms) 2024-11-21T03:55:06,235 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732161306193 2024-11-21T03:55:06,237 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T03:55:06,237 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T03:55:06,238 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=fdc0a393186c,45259,1732161302603 2024-11-21T03:55:06,238 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fdc0a393186c,45259,1732161302603, state=OPEN 2024-11-21T03:55:06,241 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34369 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fdc0a393186c,45259,1732161302603, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T03:55:06,245 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:55:06,245 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(1482): Serving as fdc0a393186c,41615,1732161306095, RpcServer on fdc0a393186c/172.17.0.2:41615, sessionid=0x1015cd182610003 2024-11-21T03:55:06,245 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T03:55:06,245 DEBUG [RS:2;fdc0a393186c:41615 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fdc0a393186c,41615,1732161306095 2024-11-21T03:55:06,245 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,41615,1732161306095' 2024-11-21T03:55:06,246 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T03:55:06,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:55:06,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:55:06,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:55:06,265 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T03:55:06,265 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fdc0a393186c,45259,1732161302603 2024-11-21T03:55:06,265 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:06,265 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:06,265 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:55:06,265 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T03:55:06,265 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T03:55:06,265 DEBUG [RS:2;fdc0a393186c:41615 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fdc0a393186c,41615,1732161306095 2024-11-21T03:55:06,266 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,41615,1732161306095' 2024-11-21T03:55:06,266 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T03:55:06,266 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T03:55:06,266 DEBUG [RS:2;fdc0a393186c:41615 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T03:55:06,266 INFO [RS:2;fdc0a393186c:41615 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T03:55:06,266 INFO [RS:2;fdc0a393186c:41615 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T03:55:06,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-21T03:55:06,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fdc0a393186c,45259,1732161302603 in 225 msec 2024-11-21T03:55:06,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T03:55:06,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 401 msec 2024-11-21T03:55:06,270 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): fdc0a393186c,44887,1732161302640 had 2 regions 2024-11-21T03:55:06,271 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fdc0a393186c,44887,1732161302640, splitWal=true, meta=true, isMeta: false 2024-11-21T03:55:06,271 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,44887,1732161302640-splitting dir is empty, no logs to split. 2024-11-21T03:55:06,271 INFO [PEWorker-3 {}] master.SplitWALManager(105): fdc0a393186c,44887,1732161302640 WAL count=0, meta=false 2024-11-21T03:55:06,272 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,44887,1732161302640-splitting dir is empty, no logs to split. 2024-11-21T03:55:06,272 INFO [PEWorker-3 {}] master.SplitWALManager(105): fdc0a393186c,44887,1732161302640 WAL count=0, meta=false 2024-11-21T03:55:06,272 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if fdc0a393186c,44887,1732161302640 WAL splitting is done? wals=0, meta=false 2024-11-21T03:55:06,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732161303888, region=6868f7430ca843fd67df9932f7d3b3c9, ASSIGN}] 2024-11-21T03:55:06,274 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732161303888, region=6868f7430ca843fd67df9932f7d3b3c9, ASSIGN 2024-11-21T03:55:06,275 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732161303888, region=6868f7430ca843fd67df9932f7d3b3c9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T03:55:06,368 INFO [RS:2;fdc0a393186c:41615 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:55:06,369 INFO [RS:2;fdc0a393186c:41615 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=fdc0a393186c%2C41615%2C1732161306095, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,41615,1732161306095, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs, maxLogs=55 2024-11-21T03:55:06,392 INFO [RS:2;fdc0a393186c:41615 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/WALs/fdc0a393186c,41615,1732161306095/fdc0a393186c%2C41615%2C1732161306095.1732161306371 2024-11-21T03:55:06,392 DEBUG [RS:2;fdc0a393186c:41615 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T03:55:06,425 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(204): Hosts are {fdc0a393186c=0} racks are {/default-rack=0} 2024-11-21T03:55:06,425 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T03:55:06,426 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T03:55:06,426 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T03:55:06,426 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T03:55:06,426 INFO [fdc0a393186c:34369 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T03:55:06,426 INFO [fdc0a393186c:34369 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T03:55:06,426 DEBUG [fdc0a393186c:34369 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T03:55:06,426 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6868f7430ca843fd67df9932f7d3b3c9, regionState=OPENING, regionLocation=fdc0a393186c,45259,1732161302603 2024-11-21T03:55:06,428 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fdc0a393186c:44887 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fdc0a393186c/172.17.0.2:44887 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:55:06,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,44887,1732161302640, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,44887,1732161302640, seqNum=-1, error=java.net.ConnectException: Call to address=fdc0a393186c:44887 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fdc0a393186c/172.17.0.2:44887 2024-11-21T03:55:06,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,44887,1732161302640, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-21T03:55:06,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,44887,1732161302640, seqNum=-1 from cache 2024-11-21T03:55:06,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address fdc0a393186c:44887 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fdc0a393186c/172.17.0.2:44887 2024-11-21T03:55:06,531 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T03:55:06,532 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T03:55:06,533 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T03:55:06,534 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T03:55:06,534 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T03:55:06,534 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T03:55:06,537 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:55:06,537 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,45259,1732161302603, seqNum=-1] 2024-11-21T03:55:06,538 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:55:06,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53041, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:55:06,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732161303888, region=6868f7430ca843fd67df9932f7d3b3c9, ASSIGN because future has completed 2024-11-21T03:55:06,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6868f7430ca843fd67df9932f7d3b3c9, server=fdc0a393186c,45259,1732161302603}] 2024-11-21T03:55:06,699 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:06,700 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 6868f7430ca843fd67df9932f7d3b3c9, NAME => 'testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:55:06,702 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732161303888 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,702 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:55:06,702 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,702 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,704 INFO [StoreOpener-6868f7430ca843fd67df9932f7d3b3c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,705 INFO [StoreOpener-6868f7430ca843fd67df9932f7d3b3c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6868f7430ca843fd67df9932f7d3b3c9 columnFamilyName family 2024-11-21T03:55:06,705 DEBUG [StoreOpener-6868f7430ca843fd67df9932f7d3b3c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:55:06,708 DEBUG [StoreFileOpener-6868f7430ca843fd67df9932f7d3b3c9-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64, delay=988 ms 2024-11-21T03:55:06,710 DEBUG [StoreOpener-6868f7430ca843fd67df9932f7d3b3c9-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64 2024-11-21T03:55:06,710 INFO [StoreOpener-6868f7430ca843fd67df9932f7d3b3c9-1 {}] regionserver.HStore(327): Store=6868f7430ca843fd67df9932f7d3b3c9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:55:06,710 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,711 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,711 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,712 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,712 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,713 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,713 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 6868f7430ca843fd67df9932f7d3b3c9; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72244217, jitterRate=0.07652272284030914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T03:55:06,714 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,715 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 6868f7430ca843fd67df9932f7d3b3c9: Running coprocessor pre-open hook at 1732161306703Writing region info on filesystem at 1732161306703Initializing all the Stores at 1732161306703Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732161306703Cleaning up temporary data from old regions at 1732161306712 (+9 ms)Running coprocessor post-open hooks at 1732161306714 (+2 ms)Region opened successfully at 1732161306715 (+1 ms) 2024-11-21T03:55:06,715 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9., pid=11, masterSystemTime=1732161306695 2024-11-21T03:55:06,719 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:06,719 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:06,719 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6868f7430ca843fd67df9932f7d3b3c9, regionState=OPEN, openSeqNum=10, regionLocation=fdc0a393186c,45259,1732161302603 2024-11-21T03:55:06,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6868f7430ca843fd67df9932f7d3b3c9, server=fdc0a393186c,45259,1732161302603 because future has completed 2024-11-21T03:55:06,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-21T03:55:06,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 6868f7430ca843fd67df9932f7d3b3c9, server=fdc0a393186c,45259,1732161302603 in 180 msec 2024-11-21T03:55:06,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-21T03:55:06,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732161303888, region=6868f7430ca843fd67df9932f7d3b3c9, ASSIGN in 452 msec 2024-11-21T03:55:06,727 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server fdc0a393186c,44887,1732161302640 after splitting done 2024-11-21T03:55:06,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure fdc0a393186c,44887,1732161302640, splitWal=true, meta=true in 877 msec 2024-11-21T03:55:06,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T03:55:06,746 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:55:06,746 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:55:06,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:06,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:06,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:55:06,746 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T03:55:06,746 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=138648413, stopped=false 2024-11-21T03:55:06,746 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fdc0a393186c,34369,1732161302442 2024-11-21T03:55:06,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T03:55:06,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T03:55:06,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T03:55:06,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:06,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:06,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:06,799 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T03:55:06,799 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:55:06,800 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:55:06,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:06,801 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fdc0a393186c,45259,1732161302603' ***** 2024-11-21T03:55:06,801 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T03:55:06,801 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:55:06,801 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:55:06,801 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fdc0a393186c,41615,1732161306095' ***** 2024-11-21T03:55:06,801 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T03:55:06,801 INFO [RS:0;fdc0a393186c:45259 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T03:55:06,802 INFO [RS:2;fdc0a393186c:41615 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T03:55:06,802 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:55:06,802 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T03:55:06,802 INFO [RS:2;fdc0a393186c:41615 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.persistence 2024-11-21T03:55:06,802 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T03:55:06,802 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T03:55:06,803 INFO [RS:0;fdc0a393186c:45259 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8798b47d-a531-46e9-d601-766a7d95e9c8/bucket.persistence 2024-11-21T03:55:06,803 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T03:55:06,803 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T03:55:06,803 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T03:55:06,803 INFO [RS:0;fdc0a393186c:45259 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T03:55:06,803 INFO [RS:2;fdc0a393186c:41615 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T03:55:06,803 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T03:55:06,803 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T03:55:06,813 DEBUG [RS:0;fdc0a393186c:45259 {}] bucket.BucketCache(1487): Thread RS:0;fdc0a393186c:45259 started persisting bucket cache to file 2024-11-21T03:55:06,813 DEBUG [RS:2;fdc0a393186c:41615 {}] bucket.BucketCache(1487): Thread RS:2;fdc0a393186c:41615 started persisting bucket cache to file 2024-11-21T03:55:06,813 DEBUG [RS:0;fdc0a393186c:45259 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T03:55:06,814 DEBUG [RS:0;fdc0a393186c:45259 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T03:55:06,814 DEBUG [RS:2;fdc0a393186c:41615 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T03:55:06,814 DEBUG [RS:2;fdc0a393186c:41615 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T03:55:06,817 DEBUG [RS:0;fdc0a393186c:45259 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T03:55:06,817 DEBUG [RS:0;fdc0a393186c:45259 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732161306813 2024-11-21T03:55:06,817 DEBUG [RS:2;fdc0a393186c:41615 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-21T03:55:06,817 DEBUG [RS:2;fdc0a393186c:41615 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732161306813 2024-11-21T03:55:06,817 DEBUG [RS:2;fdc0a393186c:41615 {}] bucket.BucketCache(1509): Thread RS:2;fdc0a393186c:41615 finished persisting bucket cache to file, renaming 2024-11-21T03:55:06,817 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T03:55:06,817 INFO [RS:2;fdc0a393186c:41615 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T03:55:06,817 INFO [RS:2;fdc0a393186c:41615 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T03:55:06,817 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T03:55:06,817 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(959): stopping server fdc0a393186c,41615,1732161306095 2024-11-21T03:55:06,817 INFO [RS:2;fdc0a393186c:41615 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T03:55:06,817 INFO [RS:2;fdc0a393186c:41615 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;fdc0a393186c:41615. 2024-11-21T03:55:06,817 DEBUG [RS:2;fdc0a393186c:41615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:55:06,818 DEBUG [RS:2;fdc0a393186c:41615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:06,818 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(976): stopping server fdc0a393186c,41615,1732161306095; all regions closed. 2024-11-21T03:55:06,819 DEBUG [RS:2;fdc0a393186c:41615 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs 2024-11-21T03:55:06,819 INFO [RS:2;fdc0a393186c:41615 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fdc0a393186c%2C41615%2C1732161306095:(num 1732161306371) 2024-11-21T03:55:06,819 DEBUG [RS:2;fdc0a393186c:41615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:06,819 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:55:06,819 INFO [RS:2;fdc0a393186c:41615 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T03:55:06,820 INFO [RS:2;fdc0a393186c:41615 {}] hbase.ChoreService(370): Chore service for: regionserver/fdc0a393186c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T03:55:06,820 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T03:55:06,820 INFO [regionserver/fdc0a393186c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T03:55:06,820 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T03:55:06,820 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T03:55:06,820 INFO [RS:2;fdc0a393186c:41615 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T03:55:06,820 INFO [RS:2;fdc0a393186c:41615 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41615 2024-11-21T03:55:06,828 DEBUG [RS:0;fdc0a393186c:45259 {}] bucket.BucketCache(1509): Thread RS:0;fdc0a393186c:45259 finished persisting bucket cache to file, renaming 2024-11-21T03:55:06,828 WARN [RS:0;fdc0a393186c:45259 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-21T03:55:06,828 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T03:55:06,829 INFO [RS:0;fdc0a393186c:45259 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T03:55:06,829 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T03:55:06,829 INFO [RS:0;fdc0a393186c:45259 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T03:55:06,829 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(3091): Received CLOSE for 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,829 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(959): stopping server fdc0a393186c,45259,1732161302603 2024-11-21T03:55:06,829 INFO [RS:0;fdc0a393186c:45259 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T03:55:06,829 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T03:55:06,829 INFO [RS:0;fdc0a393186c:45259 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fdc0a393186c:45259. 2024-11-21T03:55:06,830 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6868f7430ca843fd67df9932f7d3b3c9, disabling compactions & flushes 2024-11-21T03:55:06,830 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:06,830 DEBUG [RS:0;fdc0a393186c:45259 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:55:06,830 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:06,830 DEBUG [RS:0;fdc0a393186c:45259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:06,830 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. after waiting 0 ms 2024-11-21T03:55:06,830 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:06,830 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T03:55:06,830 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T03:55:06,830 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T03:55:06,830 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T03:55:06,831 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T03:55:06,831 DEBUG [StoreFileCloser-6868f7430ca843fd67df9932f7d3b3c9-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/family/9bd016c70233446385e73f9d8c29fa64 2024-11-21T03:55:06,831 DEBUG [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(1325): Online Regions={6868f7430ca843fd67df9932f7d3b3c9=testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9., 1588230740=hbase:meta,,1.1588230740} 2024-11-21T03:55:06,831 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T03:55:06,831 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T03:55:06,831 DEBUG [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6868f7430ca843fd67df9932f7d3b3c9 2024-11-21T03:55:06,831 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T03:55:06,831 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T03:55:06,831 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T03:55:06,831 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T03:55:06,832 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-21T03:55:06,837 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/default/testBlockEvictionOnGracefulStop1732161303888/6868f7430ca843fd67df9932f7d3b3c9/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-21T03:55:06,838 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:06,838 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6868f7430ca843fd67df9932f7d3b3c9: Waiting for close lock at 1732161306829Running coprocessor pre-close hooks at 1732161306830 (+1 ms)Disabling compacts and flushes for region at 1732161306830Disabling writes for close at 1732161306830Writing region close event to WAL at 1732161306831 (+1 ms)Running coprocessor post-close hooks at 1732161306838 (+7 ms)Closed at 1732161306838 2024-11-21T03:55:06,838 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9. 2024-11-21T03:55:06,844 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2a735cdf7af8479ba28f6098b65e4c95 2024-11-21T03:55:06,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fdc0a393186c,41615,1732161306095 2024-11-21T03:55:06,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T03:55:06,846 INFO [RS:2;fdc0a393186c:41615 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T03:55:06,847 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fdc0a393186c,41615,1732161306095] 2024-11-21T03:55:06,858 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/info/2a735cdf7af8479ba28f6098b65e4c95 is 209, key is testBlockEvictionOnGracefulStop1732161303888,,1732161303893.6868f7430ca843fd67df9932f7d3b3c9./info:regioninfo/1732161306719/Put/seqid=0 2024-11-21T03:55:06,858 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/info/2a735cdf7af8479ba28f6098b65e4c95 2024-11-21T03:55:06,861 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/.tmp/info/2a735cdf7af8479ba28f6098b65e4c95 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/2a735cdf7af8479ba28f6098b65e4c95 2024-11-21T03:55:06,862 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/2a735cdf7af8479ba28f6098b65e4c95, delay=999 ms 2024-11-21T03:55:06,862 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/2a735cdf7af8479ba28f6098b65e4c95, entries=8, sequenceid=19, filesize=6.6 K 2024-11-21T03:55:06,863 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false 2024-11-21T03:55:06,864 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/c26a0da428fb413f94e5ebbc3693a8f8 2024-11-21T03:55:06,864 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fdc0a393186c,41615,1732161306095 already deleted, retry=false 2024-11-21T03:55:06,864 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/info/2a735cdf7af8479ba28f6098b65e4c95 2024-11-21T03:55:06,864 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fdc0a393186c,41615,1732161306095 expired; onlineServers=1 2024-11-21T03:55:06,865 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/ns/a1813f5907964b5f8f1a9efc69201442 2024-11-21T03:55:06,865 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/table/80d6aa6760214a35a8935313439fdc6e 2024-11-21T03:55:06,869 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-21T03:55:06,870 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T03:55:06,870 INFO [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T03:55:06,870 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732161306831Running coprocessor pre-close hooks at 1732161306831Disabling compacts and flushes for region at 1732161306831Disabling writes for close at 1732161306831Obtaining lock to block concurrent updates at 1732161306832 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732161306832Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732161306832Flushing stores of hbase:meta,,1.1588230740 at 1732161306833 (+1 ms)Flushing 1588230740/info: creating writer at 1732161306833Flushing 1588230740/info: appending metadata at 1732161306857 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732161306857Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5036b2d7: reopening flushed file at 1732161306860 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false at 1732161306863 (+3 ms)Writing region close event to WAL at 1732161306866 (+3 ms)Running coprocessor post-close hooks at 1732161306870 (+4 ms)Closed at 1732161306870 2024-11-21T03:55:06,870 DEBUG [RS_CLOSE_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T03:55:06,883 INFO [regionserver/fdc0a393186c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:55:06,900 INFO [regionserver/fdc0a393186c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:55:06,956 INFO [RS:2;fdc0a393186c:41615 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T03:55:06,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:06,957 INFO [RS:2;fdc0a393186c:41615 {}] regionserver.HRegionServer(1031): Exiting; stopping=fdc0a393186c,41615,1732161306095; zookeeper connection closed. 2024-11-21T03:55:06,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41615-0x1015cd182610003, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:06,957 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@54c8f373 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@54c8f373 2024-11-21T03:55:07,031 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(976): stopping server fdc0a393186c,45259,1732161302603; all regions closed. 2024-11-21T03:55:07,034 DEBUG [RS:0;fdc0a393186c:45259 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs 2024-11-21T03:55:07,034 INFO [RS:0;fdc0a393186c:45259 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fdc0a393186c%2C45259%2C1732161302603.meta:.meta(num 1732161306201) 2024-11-21T03:55:07,036 DEBUG [RS:0;fdc0a393186c:45259 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/oldWALs 2024-11-21T03:55:07,036 INFO [RS:0;fdc0a393186c:45259 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fdc0a393186c%2C45259%2C1732161302603:(num 1732161303319) 2024-11-21T03:55:07,036 DEBUG [RS:0;fdc0a393186c:45259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:55:07,036 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:55:07,036 INFO [RS:0;fdc0a393186c:45259 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T03:55:07,037 INFO [RS:0;fdc0a393186c:45259 {}] hbase.ChoreService(370): Chore service for: regionserver/fdc0a393186c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T03:55:07,037 INFO [RS:0;fdc0a393186c:45259 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T03:55:07,037 INFO [regionserver/fdc0a393186c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T03:55:07,037 INFO [RS:0;fdc0a393186c:45259 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45259 2024-11-21T03:55:07,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fdc0a393186c,45259,1732161302603 2024-11-21T03:55:07,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T03:55:07,055 INFO [RS:0;fdc0a393186c:45259 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T03:55:07,065 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fdc0a393186c,45259,1732161302603] 2024-11-21T03:55:07,073 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fdc0a393186c,45259,1732161302603 already deleted, retry=false 2024-11-21T03:55:07,073 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fdc0a393186c,45259,1732161302603 expired; onlineServers=0 2024-11-21T03:55:07,073 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fdc0a393186c,34369,1732161302442' ***** 2024-11-21T03:55:07,073 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T03:55:07,073 INFO [M:0;fdc0a393186c:34369 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T03:55:07,073 INFO [M:0;fdc0a393186c:34369 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T03:55:07,073 DEBUG [M:0;fdc0a393186c:34369 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T03:55:07,073 DEBUG [M:0;fdc0a393186c:34369 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T03:55:07,073 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T03:55:07,073 DEBUG [master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.large.0-1732161303026 {}] cleaner.HFileCleaner(306): Exit Thread[master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.large.0-1732161303026,5,FailOnTimeoutGroup] 2024-11-21T03:55:07,074 DEBUG [master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.small.0-1732161303027 {}] cleaner.HFileCleaner(306): Exit Thread[master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.small.0-1732161303027,5,FailOnTimeoutGroup] 2024-11-21T03:55:07,074 INFO [M:0;fdc0a393186c:34369 {}] hbase.ChoreService(370): Chore service for: master/fdc0a393186c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T03:55:07,074 INFO [M:0;fdc0a393186c:34369 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T03:55:07,074 DEBUG [M:0;fdc0a393186c:34369 {}] master.HMaster(1795): Stopping service threads 2024-11-21T03:55:07,074 INFO [M:0;fdc0a393186c:34369 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T03:55:07,074 INFO [M:0;fdc0a393186c:34369 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T03:55:07,074 INFO [M:0;fdc0a393186c:34369 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T03:55:07,074 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T03:55:07,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T03:55:07,081 DEBUG [M:0;fdc0a393186c:34369 {}] zookeeper.ZKUtil(347): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T03:55:07,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:55:07,081 WARN [M:0;fdc0a393186c:34369 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T03:55:07,082 INFO [M:0;fdc0a393186c:34369 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/.lastflushedseqids 2024-11-21T03:55:07,087 INFO [M:0;fdc0a393186c:34369 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T03:55:07,088 INFO [M:0;fdc0a393186c:34369 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T03:55:07,088 DEBUG [M:0;fdc0a393186c:34369 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T03:55:07,088 INFO [M:0;fdc0a393186c:34369 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:07,088 DEBUG [M:0;fdc0a393186c:34369 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:07,088 DEBUG [M:0;fdc0a393186c:34369 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T03:55:07,088 DEBUG [M:0;fdc0a393186c:34369 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:07,088 INFO [M:0;fdc0a393186c:34369 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-21T03:55:07,096 DEBUG [M:0;fdc0a393186c:34369 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 80d28695c0b14e409579331421459d8d 2024-11-21T03:55:07,108 DEBUG [M:0;fdc0a393186c:34369 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/80d28695c0b14e409579331421459d8d is 82, key is hbase:meta,,1/info:regioninfo/1732161306237/Put/seqid=0 2024-11-21T03:55:07,108 INFO [M:0;fdc0a393186c:34369 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/80d28695c0b14e409579331421459d8d 2024-11-21T03:55:07,115 DEBUG [M:0;fdc0a393186c:34369 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9575330bf72c42e786051bcb3e0a669b 2024-11-21T03:55:07,127 DEBUG [M:0;fdc0a393186c:34369 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9575330bf72c42e786051bcb3e0a669b is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732161304270/Put/seqid=0 2024-11-21T03:55:07,128 INFO [M:0;fdc0a393186c:34369 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9575330bf72c42e786051bcb3e0a669b 2024-11-21T03:55:07,135 DEBUG [M:0;fdc0a393186c:34369 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0155014c1058495b9156106139193e52 2024-11-21T03:55:07,150 DEBUG [M:0;fdc0a393186c:34369 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0155014c1058495b9156106139193e52 is 69, key is fdc0a393186c,41615,1732161306095/rs:state/1732161306169/Put/seqid=0 2024-11-21T03:55:07,150 INFO [M:0;fdc0a393186c:34369 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0155014c1058495b9156106139193e52 2024-11-21T03:55:07,152 INFO [M:0;fdc0a393186c:34369 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0155014c1058495b9156106139193e52 2024-11-21T03:55:07,153 DEBUG [M:0;fdc0a393186c:34369 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/80d28695c0b14e409579331421459d8d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/80d28695c0b14e409579331421459d8d 2024-11-21T03:55:07,154 INFO [M:0;fdc0a393186c:34369 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/80d28695c0b14e409579331421459d8d, entries=14, sequenceid=102, filesize=5.9 K 2024-11-21T03:55:07,155 DEBUG [M:0;fdc0a393186c:34369 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9575330bf72c42e786051bcb3e0a669b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9575330bf72c42e786051bcb3e0a669b 2024-11-21T03:55:07,156 INFO [M:0;fdc0a393186c:34369 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9575330bf72c42e786051bcb3e0a669b, entries=11, sequenceid=102, filesize=6.5 K 2024-11-21T03:55:07,156 DEBUG [M:0;fdc0a393186c:34369 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0155014c1058495b9156106139193e52 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0155014c1058495b9156106139193e52 2024-11-21T03:55:07,158 INFO [M:0;fdc0a393186c:34369 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0155014c1058495b9156106139193e52 2024-11-21T03:55:07,158 INFO [M:0;fdc0a393186c:34369 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67119ccc-33e3-662c-8ffc-30c676d7c8f2/42be385b-360c-8b36-f1ed-7bf6ee5f2ee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0155014c1058495b9156106139193e52, entries=3, sequenceid=102, filesize=5.3 K 2024-11-21T03:55:07,159 INFO [M:0;fdc0a393186c:34369 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=102, compaction requested=false 2024-11-21T03:55:07,160 INFO [M:0;fdc0a393186c:34369 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:55:07,161 DEBUG [M:0;fdc0a393186c:34369 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732161307088Disabling compacts and flushes for region at 1732161307088Disabling writes for close at 1732161307088Obtaining lock to block concurrent updates at 1732161307088Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732161307088Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732161307088Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732161307089 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732161307089Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732161307107 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732161307107Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732161307110 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732161307127 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732161307127Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732161307129 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732161307149 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732161307149Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58f7d180: reopening flushed file at 1732161307152 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@102e083d: reopening flushed file at 1732161307154 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fc4aef9: reopening flushed file at 1732161307156 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=102, compaction requested=false at 1732161307159 (+3 ms)Writing region close event to WAL at 1732161307160 (+1 ms)Closed at 1732161307160 2024-11-21T03:55:07,161 INFO [M:0;fdc0a393186c:34369 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T03:55:07,161 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T03:55:07,162 INFO [M:0;fdc0a393186c:34369 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34369 2024-11-21T03:55:07,162 INFO [M:0;fdc0a393186c:34369 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T03:55:07,165 INFO [RS:0;fdc0a393186c:45259 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T03:55:07,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:07,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45259-0x1015cd182610001, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:07,165 INFO [RS:0;fdc0a393186c:45259 {}] regionserver.HRegionServer(1031): Exiting; stopping=fdc0a393186c,45259,1732161302603; zookeeper connection closed. 2024-11-21T03:55:07,165 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1c80e94a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1c80e94a 2024-11-21T03:55:07,165 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-21T03:55:07,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:07,290 INFO [M:0;fdc0a393186c:34369 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T03:55:07,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34369-0x1015cd182610000, quorum=127.0.0.1:58906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:55:07,294 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T03:55:07,295 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T03:55:07,296 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:55:07,302 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=351 (was 373), ProcessCount=11 (was 11), AvailableMemoryMB=6675 (was 6760)