2024-11-21 21:54:25,021 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-21 21:54:25,039 main DEBUG Took 0.016058 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 21:54:25,040 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 21:54:25,040 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 21:54:25,042 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 21:54:25,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,052 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 21:54:25,067 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,069 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,069 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,070 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,070 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,071 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,072 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,072 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,073 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,074 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,074 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,074 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,075 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,075 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,076 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,076 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,076 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,077 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,078 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:54:25,079 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,079 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 21:54:25,080 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:54:25,082 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 21:54:25,084 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 21:54:25,084 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 21:54:25,086 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 21:54:25,086 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 21:54:25,094 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 21:54:25,097 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 21:54:25,098 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 21:54:25,099 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 21:54:25,099 main DEBUG createAppenders(={Console}) 2024-11-21 21:54:25,100 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-21 21:54:25,100 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-21 21:54:25,100 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-21 21:54:25,101 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 21:54:25,101 main DEBUG OutputStream closed 2024-11-21 21:54:25,101 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 21:54:25,102 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 21:54:25,102 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-21 21:54:25,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 21:54:25,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 21:54:25,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 21:54:25,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 21:54:25,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 21:54:25,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 21:54:25,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 21:54:25,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 21:54:25,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 21:54:25,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 21:54:25,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 21:54:25,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 21:54:25,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 21:54:25,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 21:54:25,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 21:54:25,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 21:54:25,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 21:54:25,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 21:54:25,199 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 21:54:25,200 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-21 21:54:25,200 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 21:54:25,201 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-21T21:54:25,213 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-21 21:54:25,216 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 21:54:25,217 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T21:54:25,437 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff 2024-11-21T21:54:25,461 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=4510 2024-11-21T21:54:25,543 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T21:54:25,633 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:25,647 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/cluster_ddd1dc22-6656-10c0-d2e9-a307771c1927, deleteOnExit=true 2024-11-21T21:54:25,791 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/cluster_ddd1dc22-6656-10c0-d2e9-a307771c1927/zookeeper_0, clientPort=53189, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/cluster_ddd1dc22-6656-10c0-d2e9-a307771c1927/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/cluster_ddd1dc22-6656-10c0-d2e9-a307771c1927/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T21:54:25,805 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53189 2024-11-21T21:54:25,806 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:25,889 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 with version=8 2024-11-21T21:54:25,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/hbase-staging 2024-11-21T21:54:25,978 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T21:54:26,236 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2a884b411602:0 server-side Connection retries=45 2024-11-21T21:54:26,245 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:26,246 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:26,253 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:54:26,253 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:26,254 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:54:26,382 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T21:54:26,465 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:54:26,501 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T21:54:26,510 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T21:54:26,514 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:54:26,551 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 94702 (auto-detected) 2024-11-21T21:54:26,552 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T21:54:26,573 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40481 2024-11-21T21:54:26,593 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40481 connecting to ZooKeeper ensemble=127.0.0.1:53189 2024-11-21T21:54:26,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:404810x0, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:54:26,708 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40481-0x10160adb82f0000 connected 2024-11-21T21:54:26,815 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:26,817 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:26,825 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:26,829 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4, hbase.cluster.distributed=false 2024-11-21T21:54:26,852 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:54:26,857 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40481 2024-11-21T21:54:26,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40481 2024-11-21T21:54:26,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40481 2024-11-21T21:54:26,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40481 2024-11-21T21:54:26,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40481 2024-11-21T21:54:26,937 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2a884b411602:0 server-side Connection retries=45 2024-11-21T21:54:26,939 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:26,939 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:26,939 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:54:26,939 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:26,940 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:54:26,942 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T21:54:26,945 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:54:26,946 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36323 2024-11-21T21:54:26,948 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36323 connecting to ZooKeeper ensemble=127.0.0.1:53189 2024-11-21T21:54:26,948 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:26,949 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:26,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363230x0, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:54:26,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:363230x0, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:26,965 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36323-0x10160adb82f0001 connected 2024-11-21T21:54:26,975 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T21:54:26,991 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/bucket.cache 2024-11-21T21:54:26,999 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T21:54:27,000 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T21:54:27,002 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T21:54:27,004 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T21:54:27,004 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T21:54:27,009 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T21:54:27,012 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T21:54:27,013 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/bucket.persistence is complete. 2024-11-21T21:54:27,014 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T21:54:27,017 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T21:54:27,022 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:54:27,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36323 2024-11-21T21:54:27,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36323 2024-11-21T21:54:27,027 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36323 2024-11-21T21:54:27,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36323 2024-11-21T21:54:27,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36323 2024-11-21T21:54:27,058 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2a884b411602:0 server-side Connection retries=45 2024-11-21T21:54:27,059 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:27,059 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:27,059 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:54:27,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:27,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:54:27,060 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T21:54:27,061 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:54:27,062 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34559 2024-11-21T21:54:27,065 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34559 connecting to ZooKeeper ensemble=127.0.0.1:53189 2024-11-21T21:54:27,066 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:27,067 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:27,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:345590x0, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:54:27,098 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:345590x0, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:27,099 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34559-0x10160adb82f0002 connected 2024-11-21T21:54:27,099 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T21:54:27,101 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/bucket.cache 2024-11-21T21:54:27,101 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T21:54:27,101 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T21:54:27,101 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T21:54:27,102 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T21:54:27,102 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T21:54:27,102 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T21:54:27,103 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T21:54:27,103 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T21:54:27,103 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/bucket.persistence is complete. 2024-11-21T21:54:27,105 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T21:54:27,107 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:54:27,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34559 2024-11-21T21:54:27,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34559 2024-11-21T21:54:27,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34559 2024-11-21T21:54:27,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34559 2024-11-21T21:54:27,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34559 2024-11-21T21:54:27,131 DEBUG [M:0;2a884b411602:40481 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2a884b411602:40481 2024-11-21T21:54:27,132 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2a884b411602,40481,1732226066082 2024-11-21T21:54:27,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:27,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:27,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:27,150 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2a884b411602,40481,1732226066082 2024-11-21T21:54:27,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:27,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T21:54:27,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T21:54:27,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:27,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:27,191 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T21:54:27,192 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2a884b411602,40481,1732226066082 from backup master directory 2024-11-21T21:54:27,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:27,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2a884b411602,40481,1732226066082 2024-11-21T21:54:27,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:27,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:27,206 WARN [master/2a884b411602:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:54:27,207 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2a884b411602,40481,1732226066082 2024-11-21T21:54:27,209 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T21:54:27,210 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T21:54:27,217 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/hbase.id] with ID: 1844024f-c675-4d20-a420-6e85482ce2a7 2024-11-21T21:54:27,218 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/.tmp/hbase.id 2024-11-21T21:54:27,222 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/hbase.id] 2024-11-21T21:54:27,253 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:27,259 INFO [master/2a884b411602:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T21:54:27,278 INFO [master/2a884b411602:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-21T21:54:27,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:27,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:27,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:27,325 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:54:27,328 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T21:54:27,378 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:54:27,383 INFO [master/2a884b411602:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:54:27,441 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store 2024-11-21T21:54:27,460 INFO [master/2a884b411602:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T21:54:27,463 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:27,465 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T21:54:27,465 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:27,465 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:27,467 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T21:54:27,467 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:27,467 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:27,468 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732226067465Disabling compacts and flushes for region at 1732226067465Disabling writes for close at 1732226067467 (+2 ms)Writing region close event to WAL at 1732226067467Closed at 1732226067467 2024-11-21T21:54:27,468 WARN [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/.initializing 2024-11-21T21:54:27,469 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/WALs/2a884b411602,40481,1732226066082 2024-11-21T21:54:27,486 INFO [master/2a884b411602:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:54:27,502 INFO [master/2a884b411602:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2a884b411602%2C40481%2C1732226066082, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/WALs/2a884b411602,40481,1732226066082, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/oldWALs, maxLogs=10 2024-11-21T21:54:27,549 INFO [master/2a884b411602:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/WALs/2a884b411602,40481,1732226066082/2a884b411602%2C40481%2C1732226066082.1732226067508 2024-11-21T21:54:27,550 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T21:54:27,551 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:54:27,551 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:27,555 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,556 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,620 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T21:54:27,625 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:27,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:27,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T21:54:27,633 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:27,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:27,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T21:54:27,639 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:27,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:27,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T21:54:27,646 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:27,647 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:27,647 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,651 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,652 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,655 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,656 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,659 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T21:54:27,662 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:27,670 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:54:27,671 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65438371, jitterRate=-0.024892285466194153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T21:54:27,679 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732226067568Initializing all the Stores at 1732226067568Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226067569 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226067570 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226067570Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226067570Cleaning up temporary data from old regions at 1732226067656 (+86 ms)Region opened successfully at 1732226067678 (+22 ms) 2024-11-21T21:54:27,680 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T21:54:27,722 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b0a07e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T21:54:27,768 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T21:54:27,780 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T21:54:27,780 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T21:54:27,783 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T21:54:27,784 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T21:54:27,789 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T21:54:27,789 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T21:54:27,815 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T21:54:27,825 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T21:54:27,876 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T21:54:27,879 INFO [master/2a884b411602:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T21:54:27,880 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T21:54:27,888 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T21:54:27,891 INFO [master/2a884b411602:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T21:54:27,895 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T21:54:27,905 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T21:54:27,906 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T21:54:27,918 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T21:54:27,945 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T21:54:27,955 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T21:54:27,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:27,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:27,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:27,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:27,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:27,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:27,980 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2a884b411602,40481,1732226066082, sessionid=0x10160adb82f0000, setting cluster-up flag (Was=false) 2024-11-21T21:54:28,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:28,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:28,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:28,030 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T21:54:28,034 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2a884b411602,40481,1732226066082 2024-11-21T21:54:28,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:28,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:28,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:28,089 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T21:54:28,092 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2a884b411602,40481,1732226066082 2024-11-21T21:54:28,100 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T21:54:28,120 INFO [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(746): ClusterId : 1844024f-c675-4d20-a420-6e85482ce2a7 2024-11-21T21:54:28,120 INFO [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(746): ClusterId : 1844024f-c675-4d20-a420-6e85482ce2a7 2024-11-21T21:54:28,123 DEBUG [RS:1;2a884b411602:34559 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T21:54:28,123 DEBUG [RS:0;2a884b411602:36323 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T21:54:28,145 DEBUG [RS:0;2a884b411602:36323 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T21:54:28,145 DEBUG [RS:1;2a884b411602:34559 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T21:54:28,145 DEBUG [RS:0;2a884b411602:36323 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T21:54:28,145 DEBUG [RS:1;2a884b411602:34559 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T21:54:28,164 DEBUG [RS:0;2a884b411602:36323 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T21:54:28,165 DEBUG [RS:1;2a884b411602:34559 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T21:54:28,165 DEBUG [RS:0;2a884b411602:36323 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efcc7ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T21:54:28,165 DEBUG [RS:1;2a884b411602:34559 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4b718f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T21:54:28,172 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T21:54:28,180 DEBUG [RS:0;2a884b411602:36323 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2a884b411602:36323 2024-11-21T21:54:28,183 INFO [master/2a884b411602:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T21:54:28,184 INFO [RS:0;2a884b411602:36323 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T21:54:28,184 INFO [RS:0;2a884b411602:36323 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T21:54:28,184 DEBUG [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T21:54:28,186 DEBUG [RS:1;2a884b411602:34559 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2a884b411602:34559 2024-11-21T21:54:28,186 INFO [RS:1;2a884b411602:34559 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T21:54:28,186 INFO [RS:1;2a884b411602:34559 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T21:54:28,186 DEBUG [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T21:54:28,187 INFO [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(2659): reportForDuty to master=2a884b411602,40481,1732226066082 with port=36323, startcode=1732226066904 2024-11-21T21:54:28,187 INFO [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(2659): reportForDuty to master=2a884b411602,40481,1732226066082 with port=34559, startcode=1732226067057 2024-11-21T21:54:28,192 INFO [master/2a884b411602:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T21:54:28,199 DEBUG [RS:0;2a884b411602:36323 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T21:54:28,199 DEBUG [RS:1;2a884b411602:34559 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T21:54:28,198 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2a884b411602,40481,1732226066082 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T21:54:28,205 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:54:28,205 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:54:28,206 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:54:28,206 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:54:28,206 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2a884b411602:0, corePoolSize=10, maxPoolSize=10 2024-11-21T21:54:28,206 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,207 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:54:28,207 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,208 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226098208 2024-11-21T21:54:28,210 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T21:54:28,211 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T21:54:28,211 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:54:28,212 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T21:54:28,215 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T21:54:28,215 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T21:54:28,216 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T21:54:28,216 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T21:54:28,217 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:28,217 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T21:54:28,218 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,222 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T21:54:28,224 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T21:54:28,224 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T21:54:28,228 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T21:54:28,229 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T21:54:28,231 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.large.0-1732226068230,5,FailOnTimeoutGroup] 2024-11-21T21:54:28,234 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.small.0-1732226068231,5,FailOnTimeoutGroup] 2024-11-21T21:54:28,235 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,235 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T21:54:28,237 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,237 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,239 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T21:54:28,240 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 2024-11-21T21:54:28,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:28,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T21:54:28,259 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T21:54:28,260 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:28,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:28,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T21:54:28,264 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T21:54:28,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:28,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:28,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T21:54:28,270 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T21:54:28,270 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:28,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:28,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T21:54:28,273 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48305, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T21:54:28,273 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57867, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T21:54:28,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T21:54:28,276 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:28,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:28,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T21:54:28,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740 2024-11-21T21:54:28,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740 2024-11-21T21:54:28,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T21:54:28,278 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T21:54:28,278 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T21:54:28,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T21:54:28,281 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40481 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2a884b411602,34559,1732226067057 2024-11-21T21:54:28,284 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40481 {}] master.ServerManager(517): Registering regionserver=2a884b411602,34559,1732226067057 2024-11-21T21:54:28,285 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:54:28,285 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66988489, jitterRate=-0.0017937272787094116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T21:54:28,290 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732226068253Initializing all the Stores at 1732226068253Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226068253Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226068253Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226068254 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226068254Cleaning up temporary data from old regions at 1732226068278 (+24 ms)Region opened successfully at 1732226068290 (+12 ms) 2024-11-21T21:54:28,291 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T21:54:28,291 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T21:54:28,291 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T21:54:28,291 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T21:54:28,292 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T21:54:28,293 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T21:54:28,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732226068291Disabling compacts and flushes for region at 1732226068291Disabling writes for close at 1732226068291Writing region close event to WAL at 1732226068293 (+2 ms)Closed at 1732226068293 2024-11-21T21:54:28,297 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40481 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2a884b411602,36323,1732226066904 2024-11-21T21:54:28,297 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40481 {}] master.ServerManager(517): Registering regionserver=2a884b411602,36323,1732226066904 2024-11-21T21:54:28,299 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:54:28,299 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T21:54:28,301 DEBUG [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 2024-11-21T21:54:28,301 DEBUG [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 2024-11-21T21:54:28,301 DEBUG [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T21:54:28,301 DEBUG [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T21:54:28,301 DEBUG [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T21:54:28,301 DEBUG [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T21:54:28,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T21:54:28,314 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T21:54:28,316 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T21:54:28,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:54:28,357 DEBUG [RS:0;2a884b411602:36323 {}] zookeeper.ZKUtil(111): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2a884b411602,36323,1732226066904 2024-11-21T21:54:28,357 DEBUG [RS:1;2a884b411602:34559 {}] zookeeper.ZKUtil(111): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2a884b411602,34559,1732226067057 2024-11-21T21:54:28,357 WARN [RS:0;2a884b411602:36323 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:54:28,357 WARN [RS:1;2a884b411602:34559 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:54:28,357 INFO [RS:0;2a884b411602:36323 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:54:28,357 INFO [RS:1;2a884b411602:34559 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:54:28,358 DEBUG [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,34559,1732226067057 2024-11-21T21:54:28,358 DEBUG [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,36323,1732226066904 2024-11-21T21:54:28,360 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2a884b411602,36323,1732226066904] 2024-11-21T21:54:28,360 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2a884b411602,34559,1732226067057] 2024-11-21T21:54:28,386 INFO [RS:0;2a884b411602:36323 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T21:54:28,386 INFO [RS:1;2a884b411602:34559 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T21:54:28,405 INFO [RS:0;2a884b411602:36323 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T21:54:28,405 INFO [RS:1;2a884b411602:34559 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T21:54:28,410 INFO [RS:0;2a884b411602:36323 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T21:54:28,410 INFO [RS:1;2a884b411602:34559 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T21:54:28,410 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,410 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,411 INFO [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T21:54:28,411 INFO [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T21:54:28,417 INFO [RS:0;2a884b411602:36323 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T21:54:28,417 INFO [RS:1;2a884b411602:34559 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T21:54:28,419 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,419 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,420 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,420 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,420 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,420 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,420 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,420 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,421 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,421 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,421 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,421 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,421 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:54:28,421 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:54:28,421 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,421 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,422 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,422 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,422 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,422 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,422 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,422 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,422 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,422 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,422 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,422 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:28,423 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:54:28,423 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:54:28,423 DEBUG [RS:1;2a884b411602:34559 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:54:28,423 DEBUG [RS:0;2a884b411602:36323 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:54:28,424 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,424 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,424 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,425 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,425 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,425 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,36323,1732226066904-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:54:28,425 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,425 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,425 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,425 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,426 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,426 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,34559,1732226067057-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:54:28,446 INFO [RS:1;2a884b411602:34559 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T21:54:28,448 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,34559,1732226067057-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,448 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,449 INFO [RS:1;2a884b411602:34559 {}] regionserver.Replication(171): 2a884b411602,34559,1732226067057 started 2024-11-21T21:54:28,451 INFO [RS:0;2a884b411602:36323 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T21:54:28,452 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,36323,1732226066904-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,452 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,452 INFO [RS:0;2a884b411602:36323 {}] regionserver.Replication(171): 2a884b411602,36323,1732226066904 started 2024-11-21T21:54:28,467 WARN [2a884b411602:40481 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T21:54:28,468 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,468 INFO [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(1482): Serving as 2a884b411602,34559,1732226067057, RpcServer on 2a884b411602/172.17.0.2:34559, sessionid=0x10160adb82f0002 2024-11-21T21:54:28,469 DEBUG [RS:1;2a884b411602:34559 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T21:54:28,469 DEBUG [RS:1;2a884b411602:34559 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2a884b411602,34559,1732226067057 2024-11-21T21:54:28,469 DEBUG [RS:1;2a884b411602:34559 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,34559,1732226067057' 2024-11-21T21:54:28,469 DEBUG [RS:1;2a884b411602:34559 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T21:54:28,470 DEBUG [RS:1;2a884b411602:34559 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T21:54:28,471 DEBUG [RS:1;2a884b411602:34559 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T21:54:28,471 DEBUG [RS:1;2a884b411602:34559 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T21:54:28,471 DEBUG [RS:1;2a884b411602:34559 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2a884b411602,34559,1732226067057 2024-11-21T21:54:28,471 DEBUG [RS:1;2a884b411602:34559 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,34559,1732226067057' 2024-11-21T21:54:28,471 DEBUG [RS:1;2a884b411602:34559 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T21:54:28,472 DEBUG [RS:1;2a884b411602:34559 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T21:54:28,473 DEBUG [RS:1;2a884b411602:34559 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T21:54:28,473 INFO [RS:1;2a884b411602:34559 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T21:54:28,473 INFO [RS:1;2a884b411602:34559 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T21:54:28,476 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:28,476 INFO [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(1482): Serving as 2a884b411602,36323,1732226066904, RpcServer on 2a884b411602/172.17.0.2:36323, sessionid=0x10160adb82f0001 2024-11-21T21:54:28,476 DEBUG [RS:0;2a884b411602:36323 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T21:54:28,477 DEBUG [RS:0;2a884b411602:36323 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2a884b411602,36323,1732226066904 2024-11-21T21:54:28,477 DEBUG [RS:0;2a884b411602:36323 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,36323,1732226066904' 2024-11-21T21:54:28,477 DEBUG [RS:0;2a884b411602:36323 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T21:54:28,478 DEBUG [RS:0;2a884b411602:36323 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T21:54:28,479 DEBUG [RS:0;2a884b411602:36323 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T21:54:28,479 DEBUG [RS:0;2a884b411602:36323 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T21:54:28,479 DEBUG [RS:0;2a884b411602:36323 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2a884b411602,36323,1732226066904 2024-11-21T21:54:28,479 DEBUG [RS:0;2a884b411602:36323 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,36323,1732226066904' 2024-11-21T21:54:28,479 DEBUG [RS:0;2a884b411602:36323 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T21:54:28,480 DEBUG [RS:0;2a884b411602:36323 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T21:54:28,481 DEBUG [RS:0;2a884b411602:36323 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T21:54:28,481 INFO [RS:0;2a884b411602:36323 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T21:54:28,481 INFO [RS:0;2a884b411602:36323 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T21:54:28,577 INFO [RS:1;2a884b411602:34559 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:54:28,578 INFO [RS:1;2a884b411602:34559 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2a884b411602%2C34559%2C1732226067057, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,34559,1732226067057, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs, maxLogs=55 2024-11-21T21:54:28,582 INFO [RS:0;2a884b411602:36323 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:54:28,583 INFO [RS:0;2a884b411602:36323 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2a884b411602%2C36323%2C1732226066904, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,36323,1732226066904, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs, maxLogs=55 2024-11-21T21:54:28,598 INFO [RS:1;2a884b411602:34559 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,34559,1732226067057/2a884b411602%2C34559%2C1732226067057.1732226068581 2024-11-21T21:54:28,599 DEBUG [RS:1;2a884b411602:34559 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T21:54:28,601 INFO [RS:0;2a884b411602:36323 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,36323,1732226066904/2a884b411602%2C36323%2C1732226066904.1732226068585 2024-11-21T21:54:28,601 DEBUG [RS:0;2a884b411602:36323 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T21:54:28,723 DEBUG [2a884b411602:40481 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T21:54:28,734 DEBUG [2a884b411602:40481 {}] balancer.BalancerClusterState(204): Hosts are {2a884b411602=0} racks are {/default-rack=0} 2024-11-21T21:54:28,737 DEBUG [2a884b411602:40481 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T21:54:28,737 DEBUG [2a884b411602:40481 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T21:54:28,737 DEBUG [2a884b411602:40481 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T21:54:28,737 DEBUG [2a884b411602:40481 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T21:54:28,737 INFO [2a884b411602:40481 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T21:54:28,738 INFO [2a884b411602:40481 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T21:54:28,738 DEBUG [2a884b411602:40481 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T21:54:28,744 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2a884b411602,34559,1732226067057 2024-11-21T21:54:28,748 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,34559,1732226067057, state=OPENING 2024-11-21T21:54:28,784 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T21:54:28,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:28,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:28,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:28,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:28,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:28,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:28,798 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T21:54:28,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,34559,1732226067057}] 2024-11-21T21:54:28,971 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T21:54:28,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55401, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T21:54:28,983 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T21:54:28,983 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:54:28,984 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T21:54:28,985 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2a884b411602%2C34559%2C1732226067057.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,34559,1732226067057, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs, maxLogs=55 2024-11-21T21:54:29,004 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,34559,1732226067057/2a884b411602%2C34559%2C1732226067057.meta.1732226068987.meta 2024-11-21T21:54:29,005 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T21:54:29,005 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:54:29,007 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T21:54:29,009 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T21:54:29,012 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T21:54:29,016 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T21:54:29,017 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:29,017 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T21:54:29,017 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T21:54:29,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T21:54:29,020 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T21:54:29,020 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:29,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:29,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T21:54:29,021 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T21:54:29,021 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:29,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:29,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T21:54:29,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T21:54:29,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:29,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:29,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T21:54:29,023 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T21:54:29,023 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:29,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:29,024 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T21:54:29,025 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740 2024-11-21T21:54:29,025 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740 2024-11-21T21:54:29,026 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T21:54:29,026 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T21:54:29,026 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T21:54:29,027 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T21:54:29,027 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67698677, jitterRate=0.008788898587226868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T21:54:29,028 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T21:54:29,029 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732226069018Writing region info on filesystem at 1732226069018Initializing all the Stores at 1732226069018Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226069018Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226069019 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226069019Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226069019Cleaning up temporary data from old regions at 1732226069026 (+7 ms)Running coprocessor post-open hooks at 1732226069028 (+2 ms)Region opened successfully at 1732226069029 (+1 ms) 2024-11-21T21:54:29,035 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732226068963 2024-11-21T21:54:29,045 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T21:54:29,045 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T21:54:29,046 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,34559,1732226067057 2024-11-21T21:54:29,047 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,34559,1732226067057, state=OPEN 2024-11-21T21:54:29,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:29,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:29,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:29,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:29,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:29,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:29,078 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2a884b411602,34559,1732226067057 2024-11-21T21:54:29,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T21:54:29,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,34559,1732226067057 in 277 msec 2024-11-21T21:54:29,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T21:54:29,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 779 msec 2024-11-21T21:54:29,090 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:54:29,090 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T21:54:29,108 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:54:29,109 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2a884b411602,34559,1732226067057, seqNum=-1] 2024-11-21T21:54:29,131 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:54:29,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42813, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:54:29,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0260 sec 2024-11-21T21:54:29,158 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732226069158, completionTime=-1 2024-11-21T21:54:29,161 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-21T21:54:29,161 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T21:54:29,182 INFO [master/2a884b411602:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T21:54:29,182 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226129182 2024-11-21T21:54:29,182 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226189182 2024-11-21T21:54:29,182 INFO [master/2a884b411602:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-11-21T21:54:29,184 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T21:54:29,190 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,40481,1732226066082-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:29,190 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,40481,1732226066082-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:29,190 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,40481,1732226066082-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:29,192 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2a884b411602:40481, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:29,192 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:29,192 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:29,199 DEBUG [master/2a884b411602:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T21:54:29,220 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.013sec 2024-11-21T21:54:29,222 INFO [master/2a884b411602:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T21:54:29,224 INFO [master/2a884b411602:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T21:54:29,224 INFO [master/2a884b411602:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T21:54:29,225 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T21:54:29,225 INFO [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T21:54:29,226 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,40481,1732226066082-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:54:29,226 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,40481,1732226066082-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T21:54:29,227 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T21:54:29,228 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T21:54:29,228 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,40481,1732226066082-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:29,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71985066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:54:29,335 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T21:54:29,335 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T21:54:29,339 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2a884b411602,40481,-1 for getting cluster id 2024-11-21T21:54:29,342 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:54:29,349 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1844024f-c675-4d20-a420-6e85482ce2a7' 2024-11-21T21:54:29,352 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:54:29,352 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1844024f-c675-4d20-a420-6e85482ce2a7" 2024-11-21T21:54:29,353 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e09b09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:54:29,353 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2a884b411602,40481,-1] 2024-11-21T21:54:29,355 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:54:29,357 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:29,358 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:54:29,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e124b8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:54:29,361 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:54:29,367 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2a884b411602,34559,1732226067057, seqNum=-1] 2024-11-21T21:54:29,368 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:54:29,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:54:29,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2a884b411602,40481,1732226066082 2024-11-21T21:54:29,405 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:54:29,409 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2a884b411602,40481,1732226066082 2024-11-21T21:54:29,411 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b1bd13b 2024-11-21T21:54:29,412 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:54:29,414 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:54:29,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732226069400', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:54:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732226069400 2024-11-21T21:54:29,428 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732226069400 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:54:29,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732226069400" procId is: 4 2024-11-21T21:54:29,430 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:29,432 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732226069400 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:54:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:54:29,444 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732226069400-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ff506d8a184e84c49c39bfb560fc52b7, NAME => 'testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732226069400', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 2024-11-21T21:54:29,450 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732226069400-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:29,450 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732226069400-pool-0 {}] regionserver.HRegion(1722): Closing ff506d8a184e84c49c39bfb560fc52b7, disabling compactions & flushes 2024-11-21T21:54:29,450 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732226069400-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:29,451 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732226069400-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:29,451 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732226069400-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. after waiting 0 ms 2024-11-21T21:54:29,451 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732226069400-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:29,451 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732226069400-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:29,451 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732226069400-pool-0 {}] regionserver.HRegion(1676): Region close journal for ff506d8a184e84c49c39bfb560fc52b7: Waiting for close lock at 1732226069450Disabling compacts and flushes for region at 1732226069450Disabling writes for close at 1732226069451 (+1 ms)Writing region close event to WAL at 1732226069451Closed at 1732226069451 2024-11-21T21:54:29,452 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732226069400 execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:54:29,457 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732226069452"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226069452"}]},"ts":"1732226069452"} 2024-11-21T21:54:29,461 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:54:29,463 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732226069400 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:54:29,466 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732226069400","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732226069463"}]},"ts":"1732226069463"} 2024-11-21T21:54:29,469 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732226069400, state=ENABLING in hbase:meta 2024-11-21T21:54:29,470 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2a884b411602=0} racks are {/default-rack=0} 2024-11-21T21:54:29,471 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T21:54:29,471 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T21:54:29,471 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T21:54:29,471 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T21:54:29,471 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T21:54:29,471 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T21:54:29,471 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T21:54:29,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, ASSIGN}] 2024-11-21T21:54:29,474 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, ASSIGN 2024-11-21T21:54:29,475 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, ASSIGN; state=OFFLINE, location=2a884b411602,36323,1732226066904; forceNewPlan=false, retain=false 2024-11-21T21:54:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:54:29,628 INFO [2a884b411602:40481 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T21:54:29,629 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff506d8a184e84c49c39bfb560fc52b7, regionState=OPENING, regionLocation=2a884b411602,36323,1732226066904 2024-11-21T21:54:29,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, ASSIGN because future has completed 2024-11-21T21:54:29,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff506d8a184e84c49c39bfb560fc52b7, server=2a884b411602,36323,1732226066904}] 2024-11-21T21:54:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:54:29,790 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T21:54:29,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44291, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T21:54:29,794 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:29,795 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ff506d8a184e84c49c39bfb560fc52b7, NAME => 'testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:54:29,795 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732226069400 ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,795 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:29,795 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,796 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,797 INFO [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,800 INFO [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff506d8a184e84c49c39bfb560fc52b7 columnFamilyName family 2024-11-21T21:54:29,800 DEBUG [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:29,800 INFO [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] regionserver.HStore(327): Store=ff506d8a184e84c49c39bfb560fc52b7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:29,801 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,802 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,802 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,802 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,802 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,803 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,808 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:54:29,809 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ff506d8a184e84c49c39bfb560fc52b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66359903, jitterRate=-0.011160388588905334}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T21:54:29,809 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:29,810 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ff506d8a184e84c49c39bfb560fc52b7: Running coprocessor pre-open hook at 1732226069796Writing region info on filesystem at 1732226069796Initializing all the Stores at 1732226069796Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226069796Cleaning up temporary data from old regions at 1732226069802 (+6 ms)Running coprocessor post-open hooks at 1732226069809 (+7 ms)Region opened successfully at 1732226069810 (+1 ms) 2024-11-21T21:54:29,811 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7., pid=6, masterSystemTime=1732226069790 2024-11-21T21:54:29,814 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:29,814 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:29,815 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff506d8a184e84c49c39bfb560fc52b7, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,36323,1732226066904 2024-11-21T21:54:29,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff506d8a184e84c49c39bfb560fc52b7, server=2a884b411602,36323,1732226066904 because future has completed 2024-11-21T21:54:29,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T21:54:29,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ff506d8a184e84c49c39bfb560fc52b7, server=2a884b411602,36323,1732226066904 in 183 msec 2024-11-21T21:54:29,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T21:54:29,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, ASSIGN in 349 msec 2024-11-21T21:54:29,823 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732226069400 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:54:29,824 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732226069400","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732226069823"}]},"ts":"1732226069823"} 2024-11-21T21:54:29,826 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732226069400, state=ENABLED in hbase:meta 2024-11-21T21:54:29,827 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732226069400 execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:54:29,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732226069400 in 405 msec 2024-11-21T21:54:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:54:30,068 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732226069400 get assigned. Timeout = 60000ms 2024-11-21T21:54:30,068 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732226069400 completed 2024-11-21T21:54:30,070 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T21:54:30,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732226069400 assigned to meta. Checking AM states. 2024-11-21T21:54:30,076 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T21:54:30,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732226069400 assigned. 2024-11-21T21:54:30,087 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732226069400', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7., hostname=2a884b411602,36323,1732226066904, seqNum=2] 2024-11-21T21:54:30,089 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:54:30,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:54:30,101 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ff506d8a184e84c49c39bfb560fc52b7 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-21T21:54:30,127 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ecc42a1211294db5b1febfae914c1a38 2024-11-21T21:54:30,163 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/.tmp/family/ecc42a1211294db5b1febfae914c1a38 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-21T21:54:30,167 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/.tmp/family/ecc42a1211294db5b1febfae914c1a38 2024-11-21T21:54:30,197 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/.tmp/family/ecc42a1211294db5b1febfae914c1a38 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38 2024-11-21T21:54:30,198 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38, delay=961 ms 2024-11-21T21:54:30,200 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38, entries=2, sequenceid=6, filesize=4.9 K 2024-11-21T21:54:30,208 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for ff506d8a184e84c49c39bfb560fc52b7 in 105ms, sequenceid=6, compaction requested=false 2024-11-21T21:54:30,209 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732226069400' 2024-11-21T21:54:30,211 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ff506d8a184e84c49c39bfb560fc52b7: 2024-11-21T21:54:31,161 DEBUG [hfile-prefetch-1732226070198 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ecc42a1211294db5b1febfae914c1a38_0. 2024-11-21T21:54:31,165 DEBUG [hfile-prefetch-1732226070198 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ecc42a1211294db5b1febfae914c1a38_117. 2024-11-21T21:54:31,165 DEBUG [hfile-prefetch-1732226070198 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38, with total blocks 2, and data blocks 1 2024-11-21T21:54:31,166 DEBUG [hfile-prefetch-1732226070198 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-21T21:54:31,166 DEBUG [hfile-prefetch-1732226070198 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-21T21:54:31,167 DEBUG [hfile-prefetch-1732226070198 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38. Will sleep 100ms and try the verification again. 2024-11-21T21:54:31,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=ff506d8a184e84c49c39bfb560fc52b7, source=2a884b411602,36323,1732226066904, destination=2a884b411602,34559,1732226067057, warming up region on 2a884b411602,34559,1732226067057 2024-11-21T21:54:31,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=ff506d8a184e84c49c39bfb560fc52b7, source=2a884b411602,36323,1732226066904, destination=2a884b411602,34559,1732226067057, running balancer 2024-11-21T21:54:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, REOPEN/MOVE 2024-11-21T21:54:31,228 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, REOPEN/MOVE 2024-11-21T21:54:31,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34559 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34559 {}] regionserver.HRegion(7855): Warmup {ENCODED => ff506d8a184e84c49c39bfb560fc52b7, NAME => 'testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:54:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34559 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:31,230 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ff506d8a184e84c49c39bfb560fc52b7, regionState=CLOSING, regionLocation=2a884b411602,36323,1732226066904 2024-11-21T21:54:31,230 INFO [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,231 INFO [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff506d8a184e84c49c39bfb560fc52b7 columnFamilyName family 2024-11-21T21:54:31,231 DEBUG [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:31,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, REOPEN/MOVE because future has completed 2024-11-21T21:54:31,233 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-21T21:54:31,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ff506d8a184e84c49c39bfb560fc52b7, server=2a884b411602,36323,1732226066904}] 2024-11-21T21:54:31,234 DEBUG [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38 2024-11-21T21:54:31,234 INFO [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] regionserver.HStore(327): Store=ff506d8a184e84c49c39bfb560fc52b7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34559 {}] regionserver.HRegion(1722): Closing ff506d8a184e84c49c39bfb560fc52b7, disabling compactions & flushes 2024-11-21T21:54:31,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34559 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34559 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34559 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. after waiting 0 ms 2024-11-21T21:54:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34559 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,236 DEBUG [StoreFileCloser-ff506d8a184e84c49c39bfb560fc52b7-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38 2024-11-21T21:54:31,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34559 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34559 {}] regionserver.HRegion(1676): Region close journal for ff506d8a184e84c49c39bfb560fc52b7: Waiting for close lock at 1732226071235Disabling compacts and flushes for region at 1732226071235Disabling writes for close at 1732226071235Writing region close event to WAL at 1732226071236 (+1 ms)Closed at 1732226071236 2024-11-21T21:54:31,238 WARN [hfile-prefetch-1732226070198 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-21T21:54:31,239 DEBUG [hfile-prefetch-1732226070198 {}] hfile.PrefetchExecutor(121): Prefetch completed for ecc42a1211294db5b1febfae914c1a38 2024-11-21T21:54:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40481 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-21T21:54:31,306 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T21:54:31,306 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T21:54:31,306 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T21:54:31,306 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T21:54:31,306 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T21:54:31,306 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T21:54:31,306 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T21:54:31,306 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T21:54:31,329 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T21:54:31,329 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732226071306 2024-11-21T21:54:31,332 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-21T21:54:31,332 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732226071306 2024-11-21T21:54:31,332 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T21:54:31,342 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T21:54:31,342 WARN [bucket-cache-persister {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-21T21:54:31,395 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,395 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-21T21:54:31,396 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing ff506d8a184e84c49c39bfb560fc52b7, disabling compactions & flushes 2024-11-21T21:54:31,397 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,397 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,397 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. after waiting 0 ms 2024-11-21T21:54:31,397 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,399 DEBUG [StoreFileCloser-ff506d8a184e84c49c39bfb560fc52b7-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file ecc42a1211294db5b1febfae914c1a38, starting offset: 0, end offset: 9223372036854775807 2024-11-21T21:54:31,401 DEBUG [StoreFileCloser-ff506d8a184e84c49c39bfb560fc52b7-family-1 {}] bucket.BucketCache(856): removed key ecc42a1211294db5b1febfae914c1a38_0 from back map with offset lock 2094080 in the evict process 2024-11-21T21:54:31,406 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-21T21:54:31,407 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,407 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for ff506d8a184e84c49c39bfb560fc52b7: Waiting for close lock at 1732226071396Running coprocessor pre-close hooks at 1732226071396Disabling compacts and flushes for region at 1732226071396Disabling writes for close at 1732226071397 (+1 ms)Writing region close event to WAL at 1732226071401 (+4 ms)Running coprocessor post-close hooks at 1732226071406 (+5 ms)Closed at 1732226071407 (+1 ms) 2024-11-21T21:54:31,408 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding ff506d8a184e84c49c39bfb560fc52b7 move to 2a884b411602,34559,1732226067057 record at close sequenceid=6 2024-11-21T21:54:31,411 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,412 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ff506d8a184e84c49c39bfb560fc52b7, regionState=CLOSED 2024-11-21T21:54:31,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ff506d8a184e84c49c39bfb560fc52b7, server=2a884b411602,36323,1732226066904 because future has completed 2024-11-21T21:54:31,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T21:54:31,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure ff506d8a184e84c49c39bfb560fc52b7, server=2a884b411602,36323,1732226066904 in 183 msec 2024-11-21T21:54:31,419 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, REOPEN/MOVE; state=CLOSED, location=2a884b411602,34559,1732226067057; forceNewPlan=false, retain=false 2024-11-21T21:54:31,433 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T21:54:31,433 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T21:54:31,433 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T21:54:31,433 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T21:54:31,437 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T21:54:31,437 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732226071433 2024-11-21T21:54:31,437 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T21:54:31,443 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T21:54:31,443 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T21:54:31,459 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T21:54:31,459 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T21:54:31,465 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T21:54:31,466 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732226071443 2024-11-21T21:54:31,466 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T21:54:31,570 INFO [2a884b411602:40481 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T21:54:31,570 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ff506d8a184e84c49c39bfb560fc52b7, regionState=OPENING, regionLocation=2a884b411602,34559,1732226067057 2024-11-21T21:54:31,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, REOPEN/MOVE because future has completed 2024-11-21T21:54:31,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff506d8a184e84c49c39bfb560fc52b7, server=2a884b411602,34559,1732226067057}] 2024-11-21T21:54:31,735 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,736 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ff506d8a184e84c49c39bfb560fc52b7, NAME => 'testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:54:31,736 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732226069400 ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,737 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:31,737 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,737 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,739 INFO [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,740 INFO [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff506d8a184e84c49c39bfb560fc52b7 columnFamilyName family 2024-11-21T21:54:31,741 DEBUG [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:31,743 DEBUG [StoreFileOpener-ff506d8a184e84c49c39bfb560fc52b7-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38, delay=928 ms 2024-11-21T21:54:31,745 DEBUG [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38 2024-11-21T21:54:31,745 INFO [StoreOpener-ff506d8a184e84c49c39bfb560fc52b7-1 {}] regionserver.HStore(327): Store=ff506d8a184e84c49c39bfb560fc52b7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:31,745 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,746 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,746 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,746 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,746 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,750 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,751 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened ff506d8a184e84c49c39bfb560fc52b7; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73590332, jitterRate=0.09658139944076538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T21:54:31,751 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,751 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ff506d8a184e84c49c39bfb560fc52b7: Running coprocessor pre-open hook at 1732226071738Writing region info on filesystem at 1732226071738Initializing all the Stores at 1732226071738Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226071738Cleaning up temporary data from old regions at 1732226071746 (+8 ms)Running coprocessor post-open hooks at 1732226071751 (+5 ms)Region opened successfully at 1732226071751 2024-11-21T21:54:31,752 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7., pid=9, masterSystemTime=1732226071728 2024-11-21T21:54:31,755 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,755 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,756 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ff506d8a184e84c49c39bfb560fc52b7, regionState=OPEN, openSeqNum=10, regionLocation=2a884b411602,34559,1732226067057 2024-11-21T21:54:31,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff506d8a184e84c49c39bfb560fc52b7, server=2a884b411602,34559,1732226067057 because future has completed 2024-11-21T21:54:31,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-21T21:54:31,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure ff506d8a184e84c49c39bfb560fc52b7, server=2a884b411602,34559,1732226067057 in 186 msec 2024-11-21T21:54:31,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732226069400, region=ff506d8a184e84c49c39bfb560fc52b7, REOPEN/MOVE in 538 msec 2024-11-21T21:54:31,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T21:54:31,781 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:54:31,781 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:54:31,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:31,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:31,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:54:31,785 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T21:54:31,786 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=632300009, stopped=false 2024-11-21T21:54:31,786 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2a884b411602,40481,1732226066082 2024-11-21T21:54:31,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:31,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:31,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:31,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:31,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:31,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:31,821 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T21:54:31,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:31,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:31,823 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:54:31,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:31,823 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:54:31,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:31,824 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2a884b411602,36323,1732226066904' ***** 2024-11-21T21:54:31,825 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T21:54:31,825 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2a884b411602,34559,1732226067057' ***** 2024-11-21T21:54:31,825 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T21:54:31,825 INFO [RS:0;2a884b411602:36323 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T21:54:31,825 INFO [RS:1;2a884b411602:34559 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T21:54:31,825 INFO [RS:0;2a884b411602:36323 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/bucket.persistence 2024-11-21T21:54:31,825 INFO [RS:1;2a884b411602:34559 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/bucket.persistence 2024-11-21T21:54:31,826 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T21:54:31,826 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T21:54:31,826 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T21:54:31,826 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T21:54:31,826 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T21:54:31,826 INFO [RS:0;2a884b411602:36323 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T21:54:31,826 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T21:54:31,826 INFO [RS:1;2a884b411602:34559 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T21:54:31,826 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T21:54:31,826 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T21:54:31,828 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T21:54:31,828 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T21:54:31,829 INFO [regionserver/2a884b411602:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:54:31,830 INFO [regionserver/2a884b411602:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:54:31,836 DEBUG [RS:0;2a884b411602:36323 {}] bucket.BucketCache(1487): Thread RS:0;2a884b411602:36323 started persisting bucket cache to file 2024-11-21T21:54:31,836 DEBUG [RS:1;2a884b411602:34559 {}] bucket.BucketCache(1487): Thread RS:1;2a884b411602:34559 started persisting bucket cache to file 2024-11-21T21:54:31,836 DEBUG [RS:0;2a884b411602:36323 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T21:54:31,836 DEBUG [RS:0;2a884b411602:36323 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T21:54:31,836 DEBUG [RS:1;2a884b411602:34559 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T21:54:31,836 DEBUG [RS:1;2a884b411602:34559 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T21:54:31,839 DEBUG [RS:1;2a884b411602:34559 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T21:54:31,839 DEBUG [RS:0;2a884b411602:36323 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T21:54:31,839 DEBUG [RS:1;2a884b411602:34559 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732226071836 2024-11-21T21:54:31,839 DEBUG [RS:0;2a884b411602:36323 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732226071836 2024-11-21T21:54:31,839 DEBUG [RS:0;2a884b411602:36323 {}] bucket.BucketCache(1509): Thread RS:0;2a884b411602:36323 finished persisting bucket cache to file, renaming 2024-11-21T21:54:31,840 INFO [RS:0;2a884b411602:36323 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T21:54:31,840 INFO [RS:0;2a884b411602:36323 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T21:54:31,840 INFO [RS:0;2a884b411602:36323 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T21:54:31,840 INFO [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(959): stopping server 2a884b411602,36323,1732226066904 2024-11-21T21:54:31,840 INFO [RS:0;2a884b411602:36323 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:54:31,840 INFO [RS:0;2a884b411602:36323 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2a884b411602:36323. 2024-11-21T21:54:31,840 DEBUG [RS:0;2a884b411602:36323 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:54:31,840 DEBUG [RS:0;2a884b411602:36323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:31,841 INFO [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(976): stopping server 2a884b411602,36323,1732226066904; all regions closed. 2024-11-21T21:54:31,845 DEBUG [RS:0;2a884b411602:36323 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs 2024-11-21T21:54:31,845 INFO [RS:0;2a884b411602:36323 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2a884b411602%2C36323%2C1732226066904:(num 1732226068585) 2024-11-21T21:54:31,845 DEBUG [RS:0;2a884b411602:36323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:31,845 INFO [RS:0;2a884b411602:36323 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:54:31,845 INFO [RS:0;2a884b411602:36323 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:54:31,845 INFO [RS:0;2a884b411602:36323 {}] hbase.ChoreService(370): Chore service for: regionserver/2a884b411602:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T21:54:31,846 INFO [regionserver/2a884b411602:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:54:31,846 INFO [RS:0;2a884b411602:36323 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T21:54:31,846 INFO [RS:0;2a884b411602:36323 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T21:54:31,846 INFO [RS:0;2a884b411602:36323 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T21:54:31,846 INFO [RS:0;2a884b411602:36323 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:54:31,846 INFO [RS:0;2a884b411602:36323 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36323 2024-11-21T21:54:31,851 DEBUG [RS:1;2a884b411602:34559 {}] bucket.BucketCache(1509): Thread RS:1;2a884b411602:34559 finished persisting bucket cache to file, renaming 2024-11-21T21:54:31,851 WARN [RS:1;2a884b411602:34559 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-21T21:54:31,851 INFO [RS:1;2a884b411602:34559 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T21:54:31,851 INFO [RS:1;2a884b411602:34559 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T21:54:31,851 INFO [RS:1;2a884b411602:34559 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T21:54:31,851 INFO [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(3091): Received CLOSE for ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,852 INFO [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(959): stopping server 2a884b411602,34559,1732226067057 2024-11-21T21:54:31,852 INFO [RS:1;2a884b411602:34559 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:54:31,852 INFO [RS:1;2a884b411602:34559 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2a884b411602:34559. 2024-11-21T21:54:31,852 DEBUG [RS:1;2a884b411602:34559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:54:31,852 DEBUG [RS:1;2a884b411602:34559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:31,852 INFO [RS:1;2a884b411602:34559 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T21:54:31,852 INFO [RS:1;2a884b411602:34559 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T21:54:31,852 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T21:54:31,852 INFO [RS:1;2a884b411602:34559 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T21:54:31,852 INFO [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T21:54:31,852 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff506d8a184e84c49c39bfb560fc52b7, disabling compactions & flushes 2024-11-21T21:54:31,853 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,853 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,853 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. after waiting 0 ms 2024-11-21T21:54:31,853 INFO [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T21:54:31,853 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,853 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T21:54:31,853 DEBUG [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(1325): Online Regions={ff506d8a184e84c49c39bfb560fc52b7=testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7., 1588230740=hbase:meta,,1.1588230740} 2024-11-21T21:54:31,853 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T21:54:31,853 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T21:54:31,853 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T21:54:31,853 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T21:54:31,853 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T21:54:31,853 DEBUG [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ff506d8a184e84c49c39bfb560fc52b7 2024-11-21T21:54:31,853 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-21T21:54:31,854 DEBUG [StoreFileCloser-ff506d8a184e84c49c39bfb560fc52b7-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/family/ecc42a1211294db5b1febfae914c1a38 2024-11-21T21:54:31,858 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnRegionMove1732226069400/ff506d8a184e84c49c39bfb560fc52b7/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-21T21:54:31,859 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,859 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff506d8a184e84c49c39bfb560fc52b7: Waiting for close lock at 1732226071852Running coprocessor pre-close hooks at 1732226071852Disabling compacts and flushes for region at 1732226071852Disabling writes for close at 1732226071853 (+1 ms)Writing region close event to WAL at 1732226071854 (+1 ms)Running coprocessor post-close hooks at 1732226071858 (+4 ms)Closed at 1732226071858 2024-11-21T21:54:31,859 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7. 2024-11-21T21:54:31,862 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on be326fb0f26847d58db44d2af3e5bda3 2024-11-21T21:54:31,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:54:31,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2a884b411602,36323,1732226066904 2024-11-21T21:54:31,871 INFO [RS:0;2a884b411602:36323 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:54:31,879 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2a884b411602,36323,1732226066904] 2024-11-21T21:54:31,880 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/info/be326fb0f26847d58db44d2af3e5bda3 is 205, key is testBlockEvictionOnRegionMove1732226069400,,1732226069415.ff506d8a184e84c49c39bfb560fc52b7./info:regioninfo/1732226071755/Put/seqid=0 2024-11-21T21:54:31,881 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/info/be326fb0f26847d58db44d2af3e5bda3 2024-11-21T21:54:31,887 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2a884b411602,36323,1732226066904 already deleted, retry=false 2024-11-21T21:54:31,887 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2a884b411602,36323,1732226066904 expired; onlineServers=1 2024-11-21T21:54:31,889 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 28cb4fd31c8d4dd7b5992a83a560c503 2024-11-21T21:54:31,901 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/ns/28cb4fd31c8d4dd7b5992a83a560c503 is 43, key is default/ns:d/1732226069139/Put/seqid=0 2024-11-21T21:54:31,902 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/ns/28cb4fd31c8d4dd7b5992a83a560c503 2024-11-21T21:54:31,910 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 76defa891a54401888d9a007e133de83 2024-11-21T21:54:31,923 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/table/76defa891a54401888d9a007e133de83 is 78, key is testBlockEvictionOnRegionMove1732226069400/table:state/1732226069823/Put/seqid=0 2024-11-21T21:54:31,924 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/table/76defa891a54401888d9a007e133de83 2024-11-21T21:54:31,926 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/info/be326fb0f26847d58db44d2af3e5bda3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/be326fb0f26847d58db44d2af3e5bda3 2024-11-21T21:54:31,927 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/be326fb0f26847d58db44d2af3e5bda3, delay=910 ms 2024-11-21T21:54:31,929 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/be326fb0f26847d58db44d2af3e5bda3, entries=15, sequenceid=15, filesize=7.6 K 2024-11-21T21:54:31,929 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/ns/28cb4fd31c8d4dd7b5992a83a560c503 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/28cb4fd31c8d4dd7b5992a83a560c503 2024-11-21T21:54:31,931 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/28cb4fd31c8d4dd7b5992a83a560c503, delay=926 ms 2024-11-21T21:54:31,932 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/28cb4fd31c8d4dd7b5992a83a560c503, entries=2, sequenceid=15, filesize=5.0 K 2024-11-21T21:54:31,932 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/table/76defa891a54401888d9a007e133de83 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/76defa891a54401888d9a007e133de83 2024-11-21T21:54:31,933 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/76defa891a54401888d9a007e133de83, delay=959 ms 2024-11-21T21:54:31,934 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/76defa891a54401888d9a007e133de83, entries=2, sequenceid=15, filesize=5.3 K 2024-11-21T21:54:31,935 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=15, compaction requested=false 2024-11-21T21:54:31,935 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T21:54:31,937 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/be326fb0f26847d58db44d2af3e5bda3 2024-11-21T21:54:31,937 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/28cb4fd31c8d4dd7b5992a83a560c503 2024-11-21T21:54:31,938 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/76defa891a54401888d9a007e133de83 2024-11-21T21:54:31,942 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-21T21:54:31,942 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T21:54:31,943 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T21:54:31,943 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732226071853Running coprocessor pre-close hooks at 1732226071853Disabling compacts and flushes for region at 1732226071853Disabling writes for close at 1732226071853Obtaining lock to block concurrent updates at 1732226071853Preparing flush snapshotting stores in 1588230740 at 1732226071853Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732226071854 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732226071854Flushing 1588230740/info: creating writer at 1732226071855 (+1 ms)Flushing 1588230740/info: appending metadata at 1732226071878 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732226071878Flushing 1588230740/ns: creating writer at 1732226071883 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732226071901 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732226071901Flushing 1588230740/table: creating writer at 1732226071905 (+4 ms)Flushing 1588230740/table: appending metadata at 1732226071922 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732226071922Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a60a76c: reopening flushed file at 1732226071926 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a421370: reopening flushed file at 1732226071929 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34ec8d3a: reopening flushed file at 1732226071932 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=15, compaction requested=false at 1732226071935 (+3 ms)Writing region close event to WAL at 1732226071938 (+3 ms)Running coprocessor post-close hooks at 1732226071942 (+4 ms)Closed at 1732226071943 (+1 ms) 2024-11-21T21:54:31,943 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T21:54:31,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:31,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36323-0x10160adb82f0001, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:31,980 INFO [RS:0;2a884b411602:36323 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:54:31,980 INFO [RS:0;2a884b411602:36323 {}] regionserver.HRegionServer(1031): Exiting; stopping=2a884b411602,36323,1732226066904; zookeeper connection closed. 2024-11-21T21:54:31,981 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4b456672 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4b456672 2024-11-21T21:54:32,053 INFO [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(976): stopping server 2a884b411602,34559,1732226067057; all regions closed. 2024-11-21T21:54:32,058 DEBUG [RS:1;2a884b411602:34559 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs 2024-11-21T21:54:32,058 INFO [RS:1;2a884b411602:34559 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2a884b411602%2C34559%2C1732226067057.meta:.meta(num 1732226068987) 2024-11-21T21:54:32,062 DEBUG [RS:1;2a884b411602:34559 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs 2024-11-21T21:54:32,062 INFO [RS:1;2a884b411602:34559 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2a884b411602%2C34559%2C1732226067057:(num 1732226068581) 2024-11-21T21:54:32,062 DEBUG [RS:1;2a884b411602:34559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:32,062 INFO [RS:1;2a884b411602:34559 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:54:32,063 INFO [RS:1;2a884b411602:34559 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:54:32,063 INFO [RS:1;2a884b411602:34559 {}] hbase.ChoreService(370): Chore service for: regionserver/2a884b411602:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T21:54:32,064 INFO [RS:1;2a884b411602:34559 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:54:32,064 INFO [regionserver/2a884b411602:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:54:32,064 INFO [RS:1;2a884b411602:34559 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34559 2024-11-21T21:54:32,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:54:32,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2a884b411602,34559,1732226067057 2024-11-21T21:54:32,076 INFO [RS:1;2a884b411602:34559 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:54:32,084 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2a884b411602,34559,1732226067057] 2024-11-21T21:54:32,092 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2a884b411602,34559,1732226067057 already deleted, retry=false 2024-11-21T21:54:32,092 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2a884b411602,34559,1732226067057 expired; onlineServers=0 2024-11-21T21:54:32,092 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2a884b411602,40481,1732226066082' ***** 2024-11-21T21:54:32,093 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T21:54:32,093 INFO [M:0;2a884b411602:40481 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:54:32,093 INFO [M:0;2a884b411602:40481 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:54:32,093 DEBUG [M:0;2a884b411602:40481 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T21:54:32,093 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T21:54:32,093 DEBUG [M:0;2a884b411602:40481 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T21:54:32,093 DEBUG [master/2a884b411602:0:becomeActiveMaster-HFileCleaner.large.0-1732226068230 {}] cleaner.HFileCleaner(306): Exit Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.large.0-1732226068230,5,FailOnTimeoutGroup] 2024-11-21T21:54:32,093 DEBUG [master/2a884b411602:0:becomeActiveMaster-HFileCleaner.small.0-1732226068231 {}] cleaner.HFileCleaner(306): Exit Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.small.0-1732226068231,5,FailOnTimeoutGroup] 2024-11-21T21:54:32,094 INFO [M:0;2a884b411602:40481 {}] hbase.ChoreService(370): Chore service for: master/2a884b411602:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T21:54:32,094 INFO [M:0;2a884b411602:40481 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:54:32,094 DEBUG [M:0;2a884b411602:40481 {}] master.HMaster(1795): Stopping service threads 2024-11-21T21:54:32,094 INFO [M:0;2a884b411602:40481 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T21:54:32,094 INFO [M:0;2a884b411602:40481 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T21:54:32,095 INFO [M:0;2a884b411602:40481 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T21:54:32,095 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T21:54:32,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T21:54:32,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,104 DEBUG [M:0;2a884b411602:40481 {}] zookeeper.ZKUtil(347): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T21:54:32,104 WARN [M:0;2a884b411602:40481 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T21:54:32,105 INFO [M:0;2a884b411602:40481 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/.lastflushedseqids 2024-11-21T21:54:32,115 INFO [M:0;2a884b411602:40481 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T21:54:32,115 INFO [M:0;2a884b411602:40481 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T21:54:32,115 DEBUG [M:0;2a884b411602:40481 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T21:54:32,115 INFO [M:0;2a884b411602:40481 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:32,115 DEBUG [M:0;2a884b411602:40481 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:32,115 DEBUG [M:0;2a884b411602:40481 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T21:54:32,115 DEBUG [M:0;2a884b411602:40481 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:32,116 INFO [M:0;2a884b411602:40481 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-21T21:54:32,125 DEBUG [M:0;2a884b411602:40481 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8b5ad321a7814eeba2a586a3f5305875 2024-11-21T21:54:32,139 DEBUG [M:0;2a884b411602:40481 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8b5ad321a7814eeba2a586a3f5305875 is 82, key is hbase:meta,,1/info:regioninfo/1732226069045/Put/seqid=0 2024-11-21T21:54:32,140 INFO [M:0;2a884b411602:40481 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8b5ad321a7814eeba2a586a3f5305875 2024-11-21T21:54:32,147 DEBUG [M:0;2a884b411602:40481 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f7b304602f6846e389a475a5bed66bd7 2024-11-21T21:54:32,160 DEBUG [M:0;2a884b411602:40481 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f7b304602f6846e389a475a5bed66bd7 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732226069829/Put/seqid=0 2024-11-21T21:54:32,161 INFO [M:0;2a884b411602:40481 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f7b304602f6846e389a475a5bed66bd7 2024-11-21T21:54:32,168 DEBUG [M:0;2a884b411602:40481 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b491dc2316f84436a13d38eabf2abb81 2024-11-21T21:54:32,179 DEBUG [M:0;2a884b411602:40481 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b491dc2316f84436a13d38eabf2abb81 is 69, key is 2a884b411602,34559,1732226067057/rs:state/1732226068288/Put/seqid=0 2024-11-21T21:54:32,180 INFO [M:0;2a884b411602:40481 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b491dc2316f84436a13d38eabf2abb81 2024-11-21T21:54:32,182 DEBUG [M:0;2a884b411602:40481 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8b5ad321a7814eeba2a586a3f5305875 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8b5ad321a7814eeba2a586a3f5305875 2024-11-21T21:54:32,183 INFO [M:0;2a884b411602:40481 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8b5ad321a7814eeba2a586a3f5305875, entries=8, sequenceid=84, filesize=5.5 K 2024-11-21T21:54:32,184 DEBUG [M:0;2a884b411602:40481 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f7b304602f6846e389a475a5bed66bd7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f7b304602f6846e389a475a5bed66bd7 2024-11-21T21:54:32,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:32,184 INFO [RS:1;2a884b411602:34559 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:54:32,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34559-0x10160adb82f0002, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:32,184 INFO [RS:1;2a884b411602:34559 {}] regionserver.HRegionServer(1031): Exiting; stopping=2a884b411602,34559,1732226067057; zookeeper connection closed. 2024-11-21T21:54:32,184 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@143fca41 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@143fca41 2024-11-21T21:54:32,185 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-21T21:54:32,185 INFO [M:0;2a884b411602:40481 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f7b304602f6846e389a475a5bed66bd7, entries=9, sequenceid=84, filesize=6.5 K 2024-11-21T21:54:32,186 DEBUG [M:0;2a884b411602:40481 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b491dc2316f84436a13d38eabf2abb81 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b491dc2316f84436a13d38eabf2abb81 2024-11-21T21:54:32,187 INFO [M:0;2a884b411602:40481 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b491dc2316f84436a13d38eabf2abb81, entries=2, sequenceid=84, filesize=5.1 K 2024-11-21T21:54:32,188 INFO [M:0;2a884b411602:40481 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=84, compaction requested=false 2024-11-21T21:54:32,190 INFO [M:0;2a884b411602:40481 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:32,190 DEBUG [M:0;2a884b411602:40481 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732226072115Disabling compacts and flushes for region at 1732226072115Disabling writes for close at 1732226072115Obtaining lock to block concurrent updates at 1732226072116 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732226072116Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732226072116Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732226072116Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732226072116Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732226072139 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732226072139Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732226072142 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732226072159 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732226072159Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732226072162 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732226072179 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732226072179Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78a55e7f: reopening flushed file at 1732226072181 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@653bb434: reopening flushed file at 1732226072183 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e7a69cd: reopening flushed file at 1732226072186 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=84, compaction requested=false at 1732226072188 (+2 ms)Writing region close event to WAL at 1732226072190 (+2 ms)Closed at 1732226072190 2024-11-21T21:54:32,192 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:54:32,192 INFO [M:0;2a884b411602:40481 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T21:54:32,192 INFO [M:0;2a884b411602:40481 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40481 2024-11-21T21:54:32,193 INFO [M:0;2a884b411602:40481 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:54:32,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:32,329 INFO [M:0;2a884b411602:40481 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:54:32,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40481-0x10160adb82f0000, quorum=127.0.0.1:53189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:32,335 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T21:54:32,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T21:54:32,356 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:32,363 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2a884b411602:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732226070198 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2a884b411602:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2a884b411602:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732226071928 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2a884b411602:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732226071743 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732226071931 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 418), ProcessCount=11 (was 11), AvailableMemoryMB=4115 (was 4510) 2024-11-21T21:54:32,369 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=4114 2024-11-21T21:54:32,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/hadoop-log-dir so I do NOT create it in target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236 2024-11-21T21:54:32,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/hadoop-tmp-dir so I do NOT create it in target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236 2024-11-21T21:54:32,369 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:32,372 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/cluster_202e5853-f502-faa6-3c85-f2bed47bc46a, deleteOnExit=true 2024-11-21T21:54:32,374 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/cluster_202e5853-f502-faa6-3c85-f2bed47bc46a/zookeeper_0, clientPort=53043, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/cluster_202e5853-f502-faa6-3c85-f2bed47bc46a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/cluster_202e5853-f502-faa6-3c85-f2bed47bc46a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T21:54:32,375 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53043 2024-11-21T21:54:32,386 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 with version=8 2024-11-21T21:54:32,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/hbase-staging 2024-11-21T21:54:32,389 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2a884b411602:0 server-side Connection retries=45 2024-11-21T21:54:32,389 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:32,390 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:32,390 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:54:32,390 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:32,390 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:54:32,390 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T21:54:32,390 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:54:32,391 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35339 2024-11-21T21:54:32,393 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35339 connecting to ZooKeeper ensemble=127.0.0.1:53043 2024-11-21T21:54:32,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:353390x0, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:54:32,433 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35339-0x10160add1fb0000 connected 2024-11-21T21:54:32,514 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:32,515 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:32,517 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:32,517 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4, hbase.cluster.distributed=false 2024-11-21T21:54:32,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:54:32,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35339 2024-11-21T21:54:32,522 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35339 2024-11-21T21:54:32,522 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35339 2024-11-21T21:54:32,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35339 2024-11-21T21:54:32,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35339 2024-11-21T21:54:32,542 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2a884b411602:0 server-side Connection retries=45 2024-11-21T21:54:32,542 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:32,542 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:32,542 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:54:32,542 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:32,542 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:54:32,542 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T21:54:32,542 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:54:32,543 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40247 2024-11-21T21:54:32,545 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40247 connecting to ZooKeeper ensemble=127.0.0.1:53043 2024-11-21T21:54:32,545 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:32,545 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:32,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:402470x0, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:54:32,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:402470x0, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:32,556 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40247-0x10160add1fb0001 connected 2024-11-21T21:54:32,557 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T21:54:32,557 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.cache 2024-11-21T21:54:32,558 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T21:54:32,558 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T21:54:32,558 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T21:54:32,558 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T21:54:32,558 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T21:54:32,558 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T21:54:32,558 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T21:54:32,559 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T21:54:32,559 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.persistence is complete. 2024-11-21T21:54:32,559 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T21:54:32,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:54:32,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40247 2024-11-21T21:54:32,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40247 2024-11-21T21:54:32,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40247 2024-11-21T21:54:32,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40247 2024-11-21T21:54:32,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40247 2024-11-21T21:54:32,580 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2a884b411602:0 server-side Connection retries=45 2024-11-21T21:54:32,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:32,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:32,580 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:54:32,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:32,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:54:32,580 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T21:54:32,580 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:54:32,581 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44867 2024-11-21T21:54:32,583 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44867 connecting to ZooKeeper ensemble=127.0.0.1:53043 2024-11-21T21:54:32,583 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:32,583 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:32,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448670x0, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:54:32,598 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:32,598 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44867-0x10160add1fb0002 connected 2024-11-21T21:54:32,599 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T21:54:32,599 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.cache 2024-11-21T21:54:32,599 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T21:54:32,600 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T21:54:32,600 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T21:54:32,600 INFO [Thread-122 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T21:54:32,600 WARN [Thread-122 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T21:54:32,600 INFO [Thread-122 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T21:54:32,600 INFO [Thread-122 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.persistence is complete. 2024-11-21T21:54:32,601 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T21:54:32,601 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T21:54:32,602 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T21:54:32,604 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:54:32,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44867 2024-11-21T21:54:32,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44867 2024-11-21T21:54:32,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44867 2024-11-21T21:54:32,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44867 2024-11-21T21:54:32,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44867 2024-11-21T21:54:32,622 DEBUG [M:0;2a884b411602:35339 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2a884b411602:35339 2024-11-21T21:54:32,623 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2a884b411602,35339,1732226072389 2024-11-21T21:54:32,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:32,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:32,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:32,631 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2a884b411602,35339,1732226072389 2024-11-21T21:54:32,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T21:54:32,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T21:54:32,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,640 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T21:54:32,641 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2a884b411602,35339,1732226072389 from backup master directory 2024-11-21T21:54:32,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2a884b411602,35339,1732226072389 2024-11-21T21:54:32,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:32,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:32,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:54:32,653 WARN [master/2a884b411602:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:54:32,653 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2a884b411602,35339,1732226072389 2024-11-21T21:54:32,655 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/hbase.id] with ID: ab82a767-90c0-4fbe-93f7-e6e60d712b28 2024-11-21T21:54:32,655 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/.tmp/hbase.id 2024-11-21T21:54:32,659 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/hbase.id] 2024-11-21T21:54:32,699 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:32,699 INFO [master/2a884b411602:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T21:54:32,704 INFO [master/2a884b411602:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 5ms. 2024-11-21T21:54:32,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,747 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:54:32,749 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T21:54:32,749 INFO [master/2a884b411602:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:54:32,772 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store 2024-11-21T21:54:32,780 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:32,780 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T21:54:32,780 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:32,780 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:32,780 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T21:54:32,780 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:32,781 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:32,781 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732226072780Disabling compacts and flushes for region at 1732226072780Disabling writes for close at 1732226072780Writing region close event to WAL at 1732226072781 (+1 ms)Closed at 1732226072781 2024-11-21T21:54:32,781 WARN [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/.initializing 2024-11-21T21:54:32,781 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/WALs/2a884b411602,35339,1732226072389 2024-11-21T21:54:32,788 INFO [master/2a884b411602:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:54:32,793 INFO [master/2a884b411602:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2a884b411602%2C35339%2C1732226072389, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/WALs/2a884b411602,35339,1732226072389, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/oldWALs, maxLogs=10 2024-11-21T21:54:32,808 INFO [master/2a884b411602:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/WALs/2a884b411602,35339,1732226072389/2a884b411602%2C35339%2C1732226072389.1732226072793 2024-11-21T21:54:32,809 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T21:54:32,809 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:54:32,809 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:32,809 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,809 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,814 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T21:54:32,815 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:32,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:32,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T21:54:32,818 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:32,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:32,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T21:54:32,822 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:32,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:32,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T21:54:32,826 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:32,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:32,827 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,827 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,827 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,827 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,827 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,828 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T21:54:32,828 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:54:32,833 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:54:32,834 INFO [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60581377, jitterRate=-0.09726713597774506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T21:54:32,835 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732226072809Initializing all the Stores at 1732226072810 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226072810Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226072810Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226072810Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226072810Cleaning up temporary data from old regions at 1732226072827 (+17 ms)Region opened successfully at 1732226072834 (+7 ms) 2024-11-21T21:54:32,835 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T21:54:32,839 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cea085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T21:54:32,840 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T21:54:32,840 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T21:54:32,840 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T21:54:32,840 INFO [master/2a884b411602:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T21:54:32,841 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T21:54:32,841 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-21T21:54:32,841 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T21:54:32,843 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T21:54:32,844 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T21:54:32,872 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T21:54:32,873 INFO [master/2a884b411602:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T21:54:32,874 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T21:54:32,880 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T21:54:32,881 INFO [master/2a884b411602:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T21:54:32,882 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T21:54:32,889 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T21:54:32,890 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T21:54:32,897 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T21:54:32,900 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T21:54:32,910 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T21:54:32,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:32,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:32,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:32,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,923 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2a884b411602,35339,1732226072389, sessionid=0x10160add1fb0000, setting cluster-up flag (Was=false) 2024-11-21T21:54:32,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,964 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T21:54:32,966 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2a884b411602,35339,1732226072389 2024-11-21T21:54:32,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:32,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:33,005 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T21:54:33,007 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2a884b411602,35339,1732226072389 2024-11-21T21:54:33,008 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T21:54:33,009 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T21:54:33,009 INFO [master/2a884b411602:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T21:54:33,010 INFO [master/2a884b411602:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T21:54:33,010 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2a884b411602,35339,1732226072389 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T21:54:33,012 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:54:33,012 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:54:33,012 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:54:33,012 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2a884b411602:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:54:33,012 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2a884b411602:0, corePoolSize=10, maxPoolSize=10 2024-11-21T21:54:33,012 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,013 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:54:33,013 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,014 INFO [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(746): ClusterId : ab82a767-90c0-4fbe-93f7-e6e60d712b28 2024-11-21T21:54:33,014 INFO [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(746): ClusterId : ab82a767-90c0-4fbe-93f7-e6e60d712b28 2024-11-21T21:54:33,014 DEBUG [RS:1;2a884b411602:44867 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T21:54:33,014 DEBUG [RS:0;2a884b411602:40247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T21:54:33,014 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732226103014 2024-11-21T21:54:33,014 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:54:33,015 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T21:54:33,015 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T21:54:33,015 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T21:54:33,015 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T21:54:33,015 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T21:54:33,015 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T21:54:33,015 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T21:54:33,015 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:33,015 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T21:54:33,018 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,019 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T21:54:33,019 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T21:54:33,019 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T21:54:33,023 DEBUG [RS:0;2a884b411602:40247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T21:54:33,023 DEBUG [RS:1;2a884b411602:44867 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T21:54:33,023 DEBUG [RS:1;2a884b411602:44867 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T21:54:33,023 DEBUG [RS:0;2a884b411602:40247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T21:54:33,026 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T21:54:33,027 INFO [master/2a884b411602:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T21:54:33,027 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.large.0-1732226073027,5,FailOnTimeoutGroup] 2024-11-21T21:54:33,030 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.small.0-1732226073027,5,FailOnTimeoutGroup] 2024-11-21T21:54:33,030 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,030 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T21:54:33,030 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,030 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,031 DEBUG [RS:1;2a884b411602:44867 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T21:54:33,031 DEBUG [RS:0;2a884b411602:40247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T21:54:33,032 DEBUG [RS:0;2a884b411602:40247 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c9a8c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T21:54:33,032 DEBUG [RS:1;2a884b411602:44867 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53cd7060, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T21:54:33,044 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T21:54:33,044 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 2024-11-21T21:54:33,047 DEBUG [RS:0;2a884b411602:40247 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2a884b411602:40247 2024-11-21T21:54:33,047 INFO [RS:0;2a884b411602:40247 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T21:54:33,047 INFO [RS:0;2a884b411602:40247 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T21:54:33,047 DEBUG [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T21:54:33,048 INFO [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(2659): reportForDuty to master=2a884b411602,35339,1732226072389 with port=40247, startcode=1732226072541 2024-11-21T21:54:33,049 DEBUG [RS:0;2a884b411602:40247 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T21:54:33,049 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:33,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T21:54:33,050 DEBUG [RS:1;2a884b411602:44867 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2a884b411602:44867 2024-11-21T21:54:33,050 INFO [RS:1;2a884b411602:44867 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T21:54:33,050 INFO [RS:1;2a884b411602:44867 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T21:54:33,050 DEBUG [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T21:54:33,051 INFO [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(2659): reportForDuty to master=2a884b411602,35339,1732226072389 with port=44867, startcode=1732226072579 2024-11-21T21:54:33,052 DEBUG [RS:1;2a884b411602:44867 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T21:54:33,052 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38941, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T21:54:33,053 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35339 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2a884b411602,40247,1732226072541 2024-11-21T21:54:33,053 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35339 {}] master.ServerManager(517): Registering regionserver=2a884b411602,40247,1732226072541 2024-11-21T21:54:33,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T21:54:33,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:33,054 DEBUG [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 2024-11-21T21:54:33,054 DEBUG [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T21:54:33,054 DEBUG [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T21:54:33,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:33,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T21:54:33,055 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56445, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T21:54:33,056 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35339 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2a884b411602,44867,1732226072579 2024-11-21T21:54:33,056 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35339 {}] master.ServerManager(517): Registering regionserver=2a884b411602,44867,1732226072579 2024-11-21T21:54:33,058 DEBUG [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 2024-11-21T21:54:33,058 DEBUG [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T21:54:33,058 DEBUG [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T21:54:33,060 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T21:54:33,060 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:33,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:33,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T21:54:33,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:54:33,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T21:54:33,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:33,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:33,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T21:54:33,069 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T21:54:33,069 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:33,069 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:33,070 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T21:54:33,070 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740 2024-11-21T21:54:33,070 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740 2024-11-21T21:54:33,070 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T21:54:33,070 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T21:54:33,071 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T21:54:33,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T21:54:33,077 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:54:33,078 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74164979, jitterRate=0.10514430701732635}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T21:54:33,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732226073049Initializing all the Stores at 1732226073050 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226073050Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226073050Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226073050Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226073050Cleaning up temporary data from old regions at 1732226073070 (+20 ms)Region opened successfully at 1732226073078 (+8 ms) 2024-11-21T21:54:33,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T21:54:33,078 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T21:54:33,079 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T21:54:33,079 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T21:54:33,079 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T21:54:33,079 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T21:54:33,079 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732226073078Disabling compacts and flushes for region at 1732226073078Disabling writes for close at 1732226073079 (+1 ms)Writing region close event to WAL at 1732226073079Closed at 1732226073079 2024-11-21T21:54:33,080 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:54:33,080 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T21:54:33,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T21:54:33,082 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T21:54:33,082 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T21:54:33,099 DEBUG [RS:0;2a884b411602:40247 {}] zookeeper.ZKUtil(111): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2a884b411602,40247,1732226072541 2024-11-21T21:54:33,099 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2a884b411602,44867,1732226072579] 2024-11-21T21:54:33,099 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2a884b411602,40247,1732226072541] 2024-11-21T21:54:33,099 WARN [RS:0;2a884b411602:40247 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:54:33,099 INFO [RS:0;2a884b411602:40247 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:54:33,100 DEBUG [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,40247,1732226072541 2024-11-21T21:54:33,100 DEBUG [RS:1;2a884b411602:44867 {}] zookeeper.ZKUtil(111): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2a884b411602,44867,1732226072579 2024-11-21T21:54:33,100 WARN [RS:1;2a884b411602:44867 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:54:33,100 INFO [RS:1;2a884b411602:44867 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:54:33,100 DEBUG [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44867,1732226072579 2024-11-21T21:54:33,104 INFO [RS:1;2a884b411602:44867 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T21:54:33,104 INFO [RS:0;2a884b411602:40247 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T21:54:33,106 INFO [RS:1;2a884b411602:44867 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T21:54:33,106 INFO [RS:1;2a884b411602:44867 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T21:54:33,107 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,108 INFO [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T21:54:33,108 INFO [RS:0;2a884b411602:40247 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T21:54:33,108 INFO [RS:1;2a884b411602:44867 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T21:54:33,108 INFO [RS:0;2a884b411602:40247 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T21:54:33,108 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,108 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,108 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,109 INFO [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T21:54:33,109 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,109 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,109 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,109 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,109 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:54:33,109 INFO [RS:0;2a884b411602:40247 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T21:54:33,109 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,109 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,109 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,109 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,109 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:54:33,110 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:54:33,110 DEBUG [RS:1;2a884b411602:44867 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:54:33,110 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,110 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,111 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,111 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:33,111 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:54:33,111 DEBUG [RS:0;2a884b411602:40247 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:54:33,113 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,113 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,113 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,113 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,113 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,113 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,113 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,113 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,113 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,113 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,40247,1732226072541-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:54:33,113 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,114 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,44867,1732226072579-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:54:33,129 INFO [RS:0;2a884b411602:40247 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T21:54:33,129 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,40247,1732226072541-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,129 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,129 INFO [RS:0;2a884b411602:40247 {}] regionserver.Replication(171): 2a884b411602,40247,1732226072541 started 2024-11-21T21:54:33,129 INFO [RS:1;2a884b411602:44867 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T21:54:33,129 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,44867,1732226072579-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,129 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,129 INFO [RS:1;2a884b411602:44867 {}] regionserver.Replication(171): 2a884b411602,44867,1732226072579 started 2024-11-21T21:54:33,144 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,144 INFO [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(1482): Serving as 2a884b411602,40247,1732226072541, RpcServer on 2a884b411602/172.17.0.2:40247, sessionid=0x10160add1fb0001 2024-11-21T21:54:33,144 DEBUG [RS:0;2a884b411602:40247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T21:54:33,144 DEBUG [RS:0;2a884b411602:40247 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2a884b411602,40247,1732226072541 2024-11-21T21:54:33,144 DEBUG [RS:0;2a884b411602:40247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,40247,1732226072541' 2024-11-21T21:54:33,144 DEBUG [RS:0;2a884b411602:40247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T21:54:33,145 DEBUG [RS:0;2a884b411602:40247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T21:54:33,146 DEBUG [RS:0;2a884b411602:40247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T21:54:33,146 DEBUG [RS:0;2a884b411602:40247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T21:54:33,146 DEBUG [RS:0;2a884b411602:40247 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2a884b411602,40247,1732226072541 2024-11-21T21:54:33,146 DEBUG [RS:0;2a884b411602:40247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,40247,1732226072541' 2024-11-21T21:54:33,146 DEBUG [RS:0;2a884b411602:40247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T21:54:33,146 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,146 DEBUG [RS:0;2a884b411602:40247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T21:54:33,147 INFO [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(1482): Serving as 2a884b411602,44867,1732226072579, RpcServer on 2a884b411602/172.17.0.2:44867, sessionid=0x10160add1fb0002 2024-11-21T21:54:33,147 DEBUG [RS:1;2a884b411602:44867 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T21:54:33,147 DEBUG [RS:1;2a884b411602:44867 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2a884b411602,44867,1732226072579 2024-11-21T21:54:33,147 DEBUG [RS:1;2a884b411602:44867 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,44867,1732226072579' 2024-11-21T21:54:33,147 DEBUG [RS:1;2a884b411602:44867 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T21:54:33,147 DEBUG [RS:0;2a884b411602:40247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T21:54:33,147 INFO [RS:0;2a884b411602:40247 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T21:54:33,147 INFO [RS:0;2a884b411602:40247 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T21:54:33,148 DEBUG [RS:1;2a884b411602:44867 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T21:54:33,148 DEBUG [RS:1;2a884b411602:44867 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T21:54:33,148 DEBUG [RS:1;2a884b411602:44867 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T21:54:33,148 DEBUG [RS:1;2a884b411602:44867 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2a884b411602,44867,1732226072579 2024-11-21T21:54:33,148 DEBUG [RS:1;2a884b411602:44867 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,44867,1732226072579' 2024-11-21T21:54:33,148 DEBUG [RS:1;2a884b411602:44867 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T21:54:33,149 DEBUG [RS:1;2a884b411602:44867 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T21:54:33,150 DEBUG [RS:1;2a884b411602:44867 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T21:54:33,150 INFO [RS:1;2a884b411602:44867 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T21:54:33,150 INFO [RS:1;2a884b411602:44867 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T21:54:33,233 WARN [2a884b411602:35339 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T21:54:33,248 INFO [RS:0;2a884b411602:40247 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:54:33,249 INFO [RS:0;2a884b411602:40247 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2a884b411602%2C40247%2C1732226072541, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,40247,1732226072541, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs, maxLogs=55 2024-11-21T21:54:33,251 INFO [RS:1;2a884b411602:44867 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:54:33,252 INFO [RS:1;2a884b411602:44867 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2a884b411602%2C44867%2C1732226072579, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44867,1732226072579, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs, maxLogs=55 2024-11-21T21:54:33,270 INFO [RS:0;2a884b411602:40247 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,40247,1732226072541/2a884b411602%2C40247%2C1732226072541.1732226073251 2024-11-21T21:54:33,270 DEBUG [RS:0;2a884b411602:40247 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T21:54:33,270 INFO [RS:1;2a884b411602:44867 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44867,1732226072579/2a884b411602%2C44867%2C1732226072579.1732226073253 2024-11-21T21:54:33,271 DEBUG [RS:1;2a884b411602:44867 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T21:54:33,483 DEBUG [2a884b411602:35339 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T21:54:33,483 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(204): Hosts are {2a884b411602=0} racks are {/default-rack=0} 2024-11-21T21:54:33,484 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T21:54:33,484 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T21:54:33,484 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T21:54:33,484 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T21:54:33,484 INFO [2a884b411602:35339 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T21:54:33,484 INFO [2a884b411602:35339 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T21:54:33,484 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T21:54:33,485 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2a884b411602,44867,1732226072579 2024-11-21T21:54:33,486 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,44867,1732226072579, state=OPENING 2024-11-21T21:54:33,514 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T21:54:33,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:33,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:33,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:33,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:33,523 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T21:54:33,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:33,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:33,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,44867,1732226072579}] 2024-11-21T21:54:33,678 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T21:54:33,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44811, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T21:54:33,686 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T21:54:33,687 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:54:33,687 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T21:54:33,688 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2a884b411602%2C44867%2C1732226072579.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44867,1732226072579, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs, maxLogs=55 2024-11-21T21:54:33,705 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44867,1732226072579/2a884b411602%2C44867%2C1732226072579.meta.1732226073689.meta 2024-11-21T21:54:33,705 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T21:54:33,706 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:54:33,706 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T21:54:33,706 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T21:54:33,706 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T21:54:33,706 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T21:54:33,706 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:33,707 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T21:54:33,707 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T21:54:33,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T21:54:33,708 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T21:54:33,708 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:33,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:33,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T21:54:33,709 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T21:54:33,709 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:33,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:33,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T21:54:33,709 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T21:54:33,709 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:33,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:33,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T21:54:33,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T21:54:33,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:33,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:33,710 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T21:54:33,711 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740 2024-11-21T21:54:33,711 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740 2024-11-21T21:54:33,711 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T21:54:33,711 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T21:54:33,712 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T21:54:33,712 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T21:54:33,713 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58723448, jitterRate=-0.12495243549346924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T21:54:33,713 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T21:54:33,713 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732226073707Writing region info on filesystem at 1732226073707Initializing all the Stores at 1732226073707Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226073707Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226073707Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226073707Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226073707Cleaning up temporary data from old regions at 1732226073712 (+5 ms)Running coprocessor post-open hooks at 1732226073713 (+1 ms)Region opened successfully at 1732226073713 2024-11-21T21:54:33,714 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732226073677 2024-11-21T21:54:33,716 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T21:54:33,716 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T21:54:33,717 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,44867,1732226072579 2024-11-21T21:54:33,717 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,44867,1732226072579, state=OPEN 2024-11-21T21:54:33,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:33,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:33,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:33,752 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:33,753 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2a884b411602,44867,1732226072579 2024-11-21T21:54:33,753 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:33,753 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:33,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T21:54:33,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,44867,1732226072579 in 230 msec 2024-11-21T21:54:33,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T21:54:33,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 677 msec 2024-11-21T21:54:33,761 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:54:33,761 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T21:54:33,764 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:54:33,764 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2a884b411602,44867,1732226072579, seqNum=-1] 2024-11-21T21:54:33,765 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:54:33,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48817, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:54:33,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 763 msec 2024-11-21T21:54:33,773 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732226073773, completionTime=-1 2024-11-21T21:54:33,773 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-21T21:54:33,774 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T21:54:33,776 INFO [master/2a884b411602:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T21:54:33,777 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732226133777 2024-11-21T21:54:33,777 INFO [master/2a884b411602:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732226193777 2024-11-21T21:54:33,777 INFO [master/2a884b411602:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-21T21:54:33,777 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,35339,1732226072389-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,777 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,35339,1732226072389-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,777 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,35339,1732226072389-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,777 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2a884b411602:35339, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,777 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,778 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,781 DEBUG [master/2a884b411602:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T21:54:33,783 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.130sec 2024-11-21T21:54:33,783 INFO [master/2a884b411602:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T21:54:33,783 INFO [master/2a884b411602:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T21:54:33,783 INFO [master/2a884b411602:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T21:54:33,783 INFO [master/2a884b411602:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T21:54:33,784 INFO [master/2a884b411602:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T21:54:33,784 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,35339,1732226072389-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:54:33,784 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,35339,1732226072389-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T21:54:33,785 DEBUG [master/2a884b411602:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T21:54:33,785 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T21:54:33,785 INFO [master/2a884b411602:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,35339,1732226072389-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:33,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2beb4aff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:54:33,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2a884b411602,35339,-1 for getting cluster id 2024-11-21T21:54:33,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:54:33,818 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab82a767-90c0-4fbe-93f7-e6e60d712b28' 2024-11-21T21:54:33,819 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:54:33,819 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab82a767-90c0-4fbe-93f7-e6e60d712b28" 2024-11-21T21:54:33,819 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a39023f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:54:33,819 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2a884b411602,35339,-1] 2024-11-21T21:54:33,820 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:54:33,820 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:33,822 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:54:33,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3984566c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:54:33,825 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:54:33,827 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2a884b411602,44867,1732226072579, seqNum=-1] 2024-11-21T21:54:33,827 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:54:33,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:54:33,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2a884b411602,35339,1732226072389 2024-11-21T21:54:33,834 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:54:33,836 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 2a884b411602,35339,1732226072389 2024-11-21T21:54:33,836 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3046503c 2024-11-21T21:54:33,836 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:54:33,838 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:54:33,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732226073834', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:54:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35339 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732226073834 2024-11-21T21:54:33,841 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732226073834 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:54:33,841 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:33,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732226073834" procId is: 4 2024-11-21T21:54:33,842 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732226073834 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:54:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:54:33,852 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732226073834-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 01696101048e3c1c33f4d7d02363e332, NAME => 'testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732226073834', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 2024-11-21T21:54:33,857 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732226073834-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:33,857 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732226073834-pool-0 {}] regionserver.HRegion(1722): Closing 01696101048e3c1c33f4d7d02363e332, disabling compactions & flushes 2024-11-21T21:54:33,857 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732226073834-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:33,857 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732226073834-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:33,857 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732226073834-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. after waiting 0 ms 2024-11-21T21:54:33,858 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732226073834-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:33,858 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732226073834-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:33,858 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732226073834-pool-0 {}] regionserver.HRegion(1676): Region close journal for 01696101048e3c1c33f4d7d02363e332: Waiting for close lock at 1732226073857Disabling compacts and flushes for region at 1732226073857Disabling writes for close at 1732226073857Writing region close event to WAL at 1732226073858 (+1 ms)Closed at 1732226073858 2024-11-21T21:54:33,859 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732226073834 execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:54:33,859 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732226073859"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732226073859"}]},"ts":"1732226073859"} 2024-11-21T21:54:33,862 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:54:33,863 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732226073834 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:54:33,863 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732226073834","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732226073863"}]},"ts":"1732226073863"} 2024-11-21T21:54:33,866 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732226073834, state=ENABLING in hbase:meta 2024-11-21T21:54:33,866 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2a884b411602=0} racks are {/default-rack=0} 2024-11-21T21:54:33,866 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T21:54:33,866 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T21:54:33,866 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T21:54:33,866 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T21:54:33,866 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T21:54:33,866 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T21:54:33,867 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T21:54:33,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732226073834, region=01696101048e3c1c33f4d7d02363e332, ASSIGN}] 2024-11-21T21:54:33,868 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732226073834, region=01696101048e3c1c33f4d7d02363e332, ASSIGN 2024-11-21T21:54:33,869 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732226073834, region=01696101048e3c1c33f4d7d02363e332, ASSIGN; state=OFFLINE, location=2a884b411602,44867,1732226072579; forceNewPlan=false, retain=false 2024-11-21T21:54:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:54:34,020 INFO [2a884b411602:35339 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T21:54:34,020 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=01696101048e3c1c33f4d7d02363e332, regionState=OPENING, regionLocation=2a884b411602,44867,1732226072579 2024-11-21T21:54:34,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732226073834, region=01696101048e3c1c33f4d7d02363e332, ASSIGN because future has completed 2024-11-21T21:54:34,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01696101048e3c1c33f4d7d02363e332, server=2a884b411602,44867,1732226072579}] 2024-11-21T21:54:34,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:54:34,186 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:34,186 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 01696101048e3c1c33f4d7d02363e332, NAME => 'testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:54:34,188 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732226073834 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,188 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:34,188 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,188 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,190 INFO [StoreOpener-01696101048e3c1c33f4d7d02363e332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,194 INFO [StoreOpener-01696101048e3c1c33f4d7d02363e332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01696101048e3c1c33f4d7d02363e332 columnFamilyName family 2024-11-21T21:54:34,194 DEBUG [StoreOpener-01696101048e3c1c33f4d7d02363e332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:34,195 INFO [StoreOpener-01696101048e3c1c33f4d7d02363e332-1 {}] regionserver.HStore(327): Store=01696101048e3c1c33f4d7d02363e332/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:34,195 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,196 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,196 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,196 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,196 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,197 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,204 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:54:34,204 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 01696101048e3c1c33f4d7d02363e332; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59700575, jitterRate=-0.11039210855960846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T21:54:34,204 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:34,205 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 01696101048e3c1c33f4d7d02363e332: Running coprocessor pre-open hook at 1732226074189Writing region info on filesystem at 1732226074189Initializing all the Stores at 1732226074189Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226074190 (+1 ms)Cleaning up temporary data from old regions at 1732226074196 (+6 ms)Running coprocessor post-open hooks at 1732226074204 (+8 ms)Region opened successfully at 1732226074205 (+1 ms) 2024-11-21T21:54:34,205 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332., pid=6, masterSystemTime=1732226074181 2024-11-21T21:54:34,208 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:34,208 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:34,208 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=01696101048e3c1c33f4d7d02363e332, regionState=OPEN, openSeqNum=2, regionLocation=2a884b411602,44867,1732226072579 2024-11-21T21:54:34,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01696101048e3c1c33f4d7d02363e332, server=2a884b411602,44867,1732226072579 because future has completed 2024-11-21T21:54:34,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T21:54:34,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 01696101048e3c1c33f4d7d02363e332, server=2a884b411602,44867,1732226072579 in 186 msec 2024-11-21T21:54:34,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T21:54:34,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732226073834, region=01696101048e3c1c33f4d7d02363e332, ASSIGN in 347 msec 2024-11-21T21:54:34,217 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732226073834 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:54:34,218 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732226073834","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732226074218"}]},"ts":"1732226074218"} 2024-11-21T21:54:34,220 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732226073834, state=ENABLED in hbase:meta 2024-11-21T21:54:34,221 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732226073834 execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:54:34,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732226073834 in 382 msec 2024-11-21T21:54:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:54:34,475 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732226073834 completed 2024-11-21T21:54:34,475 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732226073834 get assigned. Timeout = 60000ms 2024-11-21T21:54:34,476 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T21:54:34,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732226073834 assigned to meta. Checking AM states. 2024-11-21T21:54:34,482 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T21:54:34,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732226073834 assigned. 2024-11-21T21:54:34,488 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732226073834', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332., hostname=2a884b411602,44867,1732226072579, seqNum=2] 2024-11-21T21:54:34,493 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 01696101048e3c1c33f4d7d02363e332 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-21T21:54:34,502 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8f9c33ae675f474cad5d225926761d33 2024-11-21T21:54:34,516 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/.tmp/family/8f9c33ae675f474cad5d225926761d33 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-21T21:54:34,517 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/.tmp/family/8f9c33ae675f474cad5d225926761d33 2024-11-21T21:54:34,519 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:54:34,520 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/.tmp/family/8f9c33ae675f474cad5d225926761d33 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33 2024-11-21T21:54:34,522 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33, delay=946 ms 2024-11-21T21:54:34,524 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33, entries=2, sequenceid=6, filesize=4.9 K 2024-11-21T21:54:34,525 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 01696101048e3c1c33f4d7d02363e332 in 32ms, sequenceid=6, compaction requested=false 2024-11-21T21:54:34,525 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732226073834' 2024-11-21T21:54:34,527 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 01696101048e3c1c33f4d7d02363e332: 2024-11-21T21:54:35,470 DEBUG [hfile-prefetch-1732226071743 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8f9c33ae675f474cad5d225926761d33_0. 2024-11-21T21:54:35,471 DEBUG [hfile-prefetch-1732226071743 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8f9c33ae675f474cad5d225926761d33_117. 2024-11-21T21:54:35,472 DEBUG [hfile-prefetch-1732226071743 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33, with total blocks 2, and data blocks 1 2024-11-21T21:54:35,473 DEBUG [hfile-prefetch-1732226071743 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-21T21:54:35,473 DEBUG [hfile-prefetch-1732226071743 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-21T21:54:35,473 DEBUG [hfile-prefetch-1732226071743 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33. Will sleep 100ms and try the verification again. 2024-11-21T21:54:35,528 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;2a884b411602:44867,5,FailOnTimeoutGroup] 2024-11-21T21:54:35,528 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2a884b411602,44867,1732226072579' ***** 2024-11-21T21:54:35,528 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-21T21:54:35,528 INFO [RS:1;2a884b411602:44867 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T21:54:35,549 INFO [RS:1;2a884b411602:44867 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.persistence 2024-11-21T21:54:35,549 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T21:54:35,549 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T21:54:35,549 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T21:54:35,549 INFO [RS:1;2a884b411602:44867 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T21:54:35,550 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T21:54:35,560 DEBUG [RS:1;2a884b411602:44867 {}] bucket.BucketCache(1487): Thread RS:1;2a884b411602:44867 started persisting bucket cache to file 2024-11-21T21:54:35,560 DEBUG [RS:1;2a884b411602:44867 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T21:54:35,560 DEBUG [RS:1;2a884b411602:44867 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T21:54:35,561 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T21:54:35,561 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T21:54:35,561 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T21:54:35,561 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T21:54:35,564 DEBUG [RS:1;2a884b411602:44867 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-21T21:54:35,564 DEBUG [RS:1;2a884b411602:44867 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732226075560 2024-11-21T21:54:35,564 DEBUG [RS:1;2a884b411602:44867 {}] bucket.BucketCache(1509): Thread RS:1;2a884b411602:44867 finished persisting bucket cache to file, renaming 2024-11-21T21:54:35,564 INFO [RS:1;2a884b411602:44867 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T21:54:35,564 INFO [RS:1;2a884b411602:44867 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T21:54:35,564 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T21:54:35,564 INFO [RS:1;2a884b411602:44867 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T21:54:35,565 INFO [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(3091): Received CLOSE for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:35,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T21:54:35,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732226075561 2024-11-21T21:54:35,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T21:54:35,565 INFO [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(959): stopping server 2a884b411602,44867,1732226072579 2024-11-21T21:54:35,565 INFO [RS:1;2a884b411602:44867 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:54:35,565 INFO [RS:1;2a884b411602:44867 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2a884b411602:44867. 2024-11-21T21:54:35,565 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T21:54:35,565 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 01696101048e3c1c33f4d7d02363e332, disabling compactions & flushes 2024-11-21T21:54:35,565 DEBUG [RS:1;2a884b411602:44867 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:54:35,565 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:35,565 DEBUG [RS:1;2a884b411602:44867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:35,565 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:35,566 INFO [RS:1;2a884b411602:44867 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T21:54:35,566 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. after waiting 0 ms 2024-11-21T21:54:35,566 INFO [RS:1;2a884b411602:44867 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T21:54:35,566 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:35,566 INFO [RS:1;2a884b411602:44867 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T21:54:35,566 INFO [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T21:54:35,566 INFO [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T21:54:35,566 DEBUG [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(1325): Online Regions={01696101048e3c1c33f4d7d02363e332=testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332., 1588230740=hbase:meta,,1.1588230740} 2024-11-21T21:54:35,566 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T21:54:35,566 DEBUG [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(1351): Waiting on 01696101048e3c1c33f4d7d02363e332, 1588230740 2024-11-21T21:54:35,566 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T21:54:35,567 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T21:54:35,567 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T21:54:35,567 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T21:54:35,567 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T21:54:35,567 DEBUG [StoreFileCloser-01696101048e3c1c33f4d7d02363e332-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33 2024-11-21T21:54:35,567 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-21T21:54:35,567 WARN [hfile-prefetch-1732226071743 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-21T21:54:35,568 DEBUG [hfile-prefetch-1732226071743 {}] hfile.PrefetchExecutor(121): Prefetch completed for 8f9c33ae675f474cad5d225926761d33 2024-11-21T21:54:35,572 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-21T21:54:35,573 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:35,573 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 01696101048e3c1c33f4d7d02363e332: Waiting for close lock at 1732226075565Running coprocessor pre-close hooks at 1732226075565Disabling compacts and flushes for region at 1732226075565Disabling writes for close at 1732226075566 (+1 ms)Writing region close event to WAL at 1732226075567 (+1 ms)Running coprocessor post-close hooks at 1732226075572 (+5 ms)Closed at 1732226075573 (+1 ms) 2024-11-21T21:54:35,573 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:35,576 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2d4cefc0770546e4a6c2c8246b1025f6 2024-11-21T21:54:35,587 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/info/2d4cefc0770546e4a6c2c8246b1025f6 is 209, key is testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332./info:regioninfo/1732226074208/Put/seqid=0 2024-11-21T21:54:35,588 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/info/2d4cefc0770546e4a6c2c8246b1025f6 2024-11-21T21:54:35,595 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 26573472c701433e84555f7c1e30558b 2024-11-21T21:54:35,606 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/ns/26573472c701433e84555f7c1e30558b is 43, key is default/ns:d/1732226073767/Put/seqid=0 2024-11-21T21:54:35,606 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/ns/26573472c701433e84555f7c1e30558b 2024-11-21T21:54:35,614 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e8a5e9c448374210b72bfd8d5183c840 2024-11-21T21:54:35,616 INFO [regionserver/2a884b411602:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:54:35,625 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/table/e8a5e9c448374210b72bfd8d5183c840 is 80, key is testBlockEvictionOnGracefulStop1732226073834/table:state/1732226074218/Put/seqid=0 2024-11-21T21:54:35,625 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/table/e8a5e9c448374210b72bfd8d5183c840 2024-11-21T21:54:35,628 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/info/2d4cefc0770546e4a6c2c8246b1025f6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/2d4cefc0770546e4a6c2c8246b1025f6 2024-11-21T21:54:35,629 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/2d4cefc0770546e4a6c2c8246b1025f6, delay=959 ms 2024-11-21T21:54:35,630 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/2d4cefc0770546e4a6c2c8246b1025f6, entries=10, sequenceid=11, filesize=7.0 K 2024-11-21T21:54:35,630 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/ns/26573472c701433e84555f7c1e30558b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/26573472c701433e84555f7c1e30558b 2024-11-21T21:54:35,631 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/26573472c701433e84555f7c1e30558b, delay=991 ms 2024-11-21T21:54:35,632 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/26573472c701433e84555f7c1e30558b, entries=2, sequenceid=11, filesize=5.0 K 2024-11-21T21:54:35,633 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/table/e8a5e9c448374210b72bfd8d5183c840 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/e8a5e9c448374210b72bfd8d5183c840 2024-11-21T21:54:35,634 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/e8a5e9c448374210b72bfd8d5183c840, delay=972 ms 2024-11-21T21:54:35,635 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/e8a5e9c448374210b72bfd8d5183c840, entries=2, sequenceid=11, filesize=5.3 K 2024-11-21T21:54:35,636 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 69ms, sequenceid=11, compaction requested=false 2024-11-21T21:54:35,636 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/2d4cefc0770546e4a6c2c8246b1025f6 2024-11-21T21:54:35,637 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/26573472c701433e84555f7c1e30558b 2024-11-21T21:54:35,637 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/e8a5e9c448374210b72bfd8d5183c840 2024-11-21T21:54:35,641 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-21T21:54:35,641 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T21:54:35,641 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T21:54:35,641 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732226075566Running coprocessor pre-close hooks at 1732226075566Disabling compacts and flushes for region at 1732226075566Disabling writes for close at 1732226075567 (+1 ms)Obtaining lock to block concurrent updates at 1732226075567Preparing flush snapshotting stores in 1588230740 at 1732226075567Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732226075568 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732226075568Flushing 1588230740/info: creating writer at 1732226075568Flushing 1588230740/info: appending metadata at 1732226075587 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732226075587Flushing 1588230740/ns: creating writer at 1732226075590 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732226075605 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732226075605Flushing 1588230740/table: creating writer at 1732226075608 (+3 ms)Flushing 1588230740/table: appending metadata at 1732226075624 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732226075624Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@154be8c3: reopening flushed file at 1732226075628 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ca84d3c: reopening flushed file at 1732226075630 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7108af8d: reopening flushed file at 1732226075632 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 69ms, sequenceid=11, compaction requested=false at 1732226075636 (+4 ms)Writing region close event to WAL at 1732226075638 (+2 ms)Running coprocessor post-close hooks at 1732226075641 (+3 ms)Closed at 1732226075641 2024-11-21T21:54:35,642 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T21:54:35,767 INFO [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(976): stopping server 2a884b411602,44867,1732226072579; all regions closed. 2024-11-21T21:54:35,769 DEBUG [RS:1;2a884b411602:44867 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs 2024-11-21T21:54:35,769 INFO [RS:1;2a884b411602:44867 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2a884b411602%2C44867%2C1732226072579.meta:.meta(num 1732226073689) 2024-11-21T21:54:35,770 DEBUG [RS:1;2a884b411602:44867 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs 2024-11-21T21:54:35,770 INFO [RS:1;2a884b411602:44867 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2a884b411602%2C44867%2C1732226072579:(num 1732226073253) 2024-11-21T21:54:35,770 DEBUG [RS:1;2a884b411602:44867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:35,770 INFO [RS:1;2a884b411602:44867 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:54:35,770 INFO [RS:1;2a884b411602:44867 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:54:35,770 INFO [RS:1;2a884b411602:44867 {}] hbase.ChoreService(370): Chore service for: regionserver/2a884b411602:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T21:54:35,771 INFO [RS:1;2a884b411602:44867 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:54:35,771 INFO [regionserver/2a884b411602:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:54:35,771 INFO [RS:1;2a884b411602:44867 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44867 2024-11-21T21:54:35,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:54:35,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2a884b411602,44867,1732226072579 2024-11-21T21:54:35,794 INFO [RS:1;2a884b411602:44867 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:54:35,802 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2a884b411602,44867,1732226072579] 2024-11-21T21:54:35,810 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2a884b411602,44867,1732226072579 already deleted, retry=false 2024-11-21T21:54:35,810 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2a884b411602,44867,1732226072579 on 2a884b411602,35339,1732226072389 2024-11-21T21:54:35,817 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2a884b411602,44867,1732226072579, splitWal=true, meta=true 2024-11-21T21:54:35,821 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 2a884b411602,44867,1732226072579 (carryingMeta=true) 2a884b411602,44867,1732226072579/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@5ba1d0cd[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-21T21:54:35,821 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2a884b411602,44867,1732226072579, splitWal=true, meta=true 2024-11-21T21:54:35,822 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 2a884b411602,44867,1732226072579, splitWal=true, meta=true, isMeta: true 2024-11-21T21:54:35,823 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44867,1732226072579-splitting 2024-11-21T21:54:35,823 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44867,1732226072579-splitting dir is empty, no logs to split. 2024-11-21T21:54:35,823 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2a884b411602,44867,1732226072579 WAL count=0, meta=true 2024-11-21T21:54:35,826 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44867,1732226072579-splitting dir is empty, no logs to split. 2024-11-21T21:54:35,826 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2a884b411602,44867,1732226072579 WAL count=0, meta=true 2024-11-21T21:54:35,826 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 2a884b411602,44867,1732226072579 WAL splitting is done? wals=0, meta=true 2024-11-21T21:54:35,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T21:54:35,828 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T21:54:35,829 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T21:54:35,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:35,902 INFO [RS:1;2a884b411602:44867 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:54:35,902 INFO [RS:1;2a884b411602:44867 {}] regionserver.HRegionServer(1031): Exiting; stopping=2a884b411602,44867,1732226072579; zookeeper connection closed. 2024-11-21T21:54:35,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44867-0x10160add1fb0002, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:35,903 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@18fbe951 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@18fbe951 2024-11-21T21:54:35,980 DEBUG [2a884b411602:35339 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T21:54:35,981 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2a884b411602,40247,1732226072541 2024-11-21T21:54:35,984 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,40247,1732226072541, state=OPENING 2024-11-21T21:54:36,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:36,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:36,022 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T21:54:36,022 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:36,022 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:36,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,40247,1732226072541}] 2024-11-21T21:54:36,049 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2a884b411602:0 server-side Connection retries=45 2024-11-21T21:54:36,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:36,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:36,050 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:54:36,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:54:36,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:54:36,050 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T21:54:36,050 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:54:36,051 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44029 2024-11-21T21:54:36,052 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44029 connecting to ZooKeeper ensemble=127.0.0.1:53043 2024-11-21T21:54:36,052 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:36,053 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:36,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:440290x0, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:54:36,064 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44029-0x10160add1fb0003 connected 2024-11-21T21:54:36,064 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-21T21:54:36,064 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-21T21:54:36,065 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T21:54:36,066 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.cache 2024-11-21T21:54:36,066 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T21:54:36,066 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T21:54:36,066 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T21:54:36,067 INFO [Thread-209 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T21:54:36,067 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T21:54:36,067 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T21:54:36,068 INFO [Thread-209 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-21T21:54:36,068 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T21:54:36,069 INFO [Thread-209 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-21T21:54:36,071 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:54:36,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44029 2024-11-21T21:54:36,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44029 2024-11-21T21:54:36,072 DEBUG [Thread-209 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-21T21:54:36,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44029 2024-11-21T21:54:36,073 INFO [Thread-209 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T21:54:36,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44029 2024-11-21T21:54:36,073 INFO [Thread-209 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-21T21:54:36,073 INFO [Thread-209 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.persistence is complete. 2024-11-21T21:54:36,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44029 2024-11-21T21:54:36,075 INFO [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(746): ClusterId : ab82a767-90c0-4fbe-93f7-e6e60d712b28 2024-11-21T21:54:36,075 DEBUG [RS:2;2a884b411602:44029 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T21:54:36,086 DEBUG [RS:2;2a884b411602:44029 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T21:54:36,086 DEBUG [RS:2;2a884b411602:44029 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T21:54:36,098 DEBUG [RS:2;2a884b411602:44029 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T21:54:36,098 DEBUG [RS:2;2a884b411602:44029 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62622e3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2a884b411602/172.17.0.2:0 2024-11-21T21:54:36,110 DEBUG [RS:2;2a884b411602:44029 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2a884b411602:44029 2024-11-21T21:54:36,110 INFO [RS:2;2a884b411602:44029 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T21:54:36,110 INFO [RS:2;2a884b411602:44029 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T21:54:36,110 DEBUG [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T21:54:36,112 INFO [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(2659): reportForDuty to master=2a884b411602,35339,1732226072389 with port=44029, startcode=1732226076049 2024-11-21T21:54:36,112 DEBUG [RS:2;2a884b411602:44029 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T21:54:36,114 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40601, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T21:54:36,115 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35339 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2a884b411602,44029,1732226076049 2024-11-21T21:54:36,115 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35339 {}] master.ServerManager(517): Registering regionserver=2a884b411602,44029,1732226076049 2024-11-21T21:54:36,116 DEBUG [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4 2024-11-21T21:54:36,116 DEBUG [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T21:54:36,116 DEBUG [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T21:54:36,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:54:36,127 DEBUG [RS:2;2a884b411602:44029 {}] zookeeper.ZKUtil(111): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2a884b411602,44029,1732226076049 2024-11-21T21:54:36,127 WARN [RS:2;2a884b411602:44029 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:54:36,127 INFO [RS:2;2a884b411602:44029 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:54:36,127 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2a884b411602,44029,1732226076049] 2024-11-21T21:54:36,127 DEBUG [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44029,1732226076049 2024-11-21T21:54:36,133 INFO [RS:2;2a884b411602:44029 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T21:54:36,135 INFO [RS:2;2a884b411602:44029 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T21:54:36,136 INFO [RS:2;2a884b411602:44029 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T21:54:36,136 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:36,136 INFO [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T21:54:36,136 INFO [RS:2;2a884b411602:44029 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T21:54:36,137 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2a884b411602:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,137 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2a884b411602:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:54:36,138 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:54:36,138 DEBUG [RS:2;2a884b411602:44029 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2a884b411602:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:54:36,138 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:36,138 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:36,138 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:36,138 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:36,138 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:36,138 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,44029,1732226076049-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:54:36,153 INFO [RS:2;2a884b411602:44029 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T21:54:36,154 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=2a884b411602,44029,1732226076049-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:36,154 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:36,154 INFO [RS:2;2a884b411602:44029 {}] regionserver.Replication(171): 2a884b411602,44029,1732226076049 started 2024-11-21T21:54:36,168 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:54:36,168 INFO [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(1482): Serving as 2a884b411602,44029,1732226076049, RpcServer on 2a884b411602/172.17.0.2:44029, sessionid=0x10160add1fb0003 2024-11-21T21:54:36,168 DEBUG [RS:2;2a884b411602:44029 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T21:54:36,169 DEBUG [RS:2;2a884b411602:44029 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2a884b411602,44029,1732226076049 2024-11-21T21:54:36,169 DEBUG [RS:2;2a884b411602:44029 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,44029,1732226076049' 2024-11-21T21:54:36,169 DEBUG [RS:2;2a884b411602:44029 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T21:54:36,169 DEBUG [RS:2;2a884b411602:44029 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T21:54:36,170 DEBUG [RS:2;2a884b411602:44029 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T21:54:36,170 DEBUG [RS:2;2a884b411602:44029 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T21:54:36,170 DEBUG [RS:2;2a884b411602:44029 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2a884b411602,44029,1732226076049 2024-11-21T21:54:36,170 DEBUG [RS:2;2a884b411602:44029 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2a884b411602,44029,1732226076049' 2024-11-21T21:54:36,170 DEBUG [RS:2;2a884b411602:44029 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T21:54:36,171 DEBUG [RS:2;2a884b411602:44029 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T21:54:36,171 DEBUG [RS:2;2a884b411602:44029 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T21:54:36,171 INFO [RS:2;2a884b411602:44029 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T21:54:36,171 INFO [RS:2;2a884b411602:44029 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T21:54:36,176 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T21:54:36,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49929, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T21:54:36,179 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T21:54:36,179 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:54:36,180 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T21:54:36,180 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2a884b411602%2C40247%2C1732226072541.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,40247,1732226072541, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs, maxLogs=55 2024-11-21T21:54:36,196 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,40247,1732226072541/2a884b411602%2C40247%2C1732226072541.meta.1732226076181.meta 2024-11-21T21:54:36,196 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T21:54:36,197 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:54:36,197 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T21:54:36,197 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T21:54:36,197 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T21:54:36,197 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T21:54:36,197 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:36,197 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T21:54:36,197 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T21:54:36,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T21:54:36,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T21:54:36,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:36,200 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/2d4cefc0770546e4a6c2c8246b1025f6, delay=993 ms 2024-11-21T21:54:36,201 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/2d4cefc0770546e4a6c2c8246b1025f6 2024-11-21T21:54:36,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:36,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T21:54:36,202 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T21:54:36,202 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:36,203 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/26573472c701433e84555f7c1e30558b, delay=907 ms 2024-11-21T21:54:36,204 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/26573472c701433e84555f7c1e30558b 2024-11-21T21:54:36,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:36,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T21:54:36,205 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T21:54:36,205 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:36,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:36,206 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T21:54:36,206 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T21:54:36,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:36,208 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/e8a5e9c448374210b72bfd8d5183c840, delay=940 ms 2024-11-21T21:54:36,208 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/e8a5e9c448374210b72bfd8d5183c840 2024-11-21T21:54:36,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:54:36,209 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T21:54:36,209 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740 2024-11-21T21:54:36,210 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740 2024-11-21T21:54:36,210 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T21:54:36,210 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T21:54:36,211 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T21:54:36,211 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T21:54:36,212 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71944835, jitterRate=0.07206158339977264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T21:54:36,212 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T21:54:36,213 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732226076198Writing region info on filesystem at 1732226076198Initializing all the Stores at 1732226076198Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226076198Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226076198Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226076198Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732226076198Cleaning up temporary data from old regions at 1732226076210 (+12 ms)Running coprocessor post-open hooks at 1732226076212 (+2 ms)Region opened successfully at 1732226076213 (+1 ms) 2024-11-21T21:54:36,214 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732226076176 2024-11-21T21:54:36,217 DEBUG [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T21:54:36,217 INFO [RS_OPEN_META-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T21:54:36,217 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=2a884b411602,40247,1732226072541 2024-11-21T21:54:36,218 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2a884b411602,40247,1732226072541, state=OPEN 2024-11-21T21:54:36,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:36,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:36,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:54:36,235 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2a884b411602,40247,1732226072541 2024-11-21T21:54:36,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:36,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:36,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:54:36,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-21T21:54:36,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2a884b411602,40247,1732226072541 in 213 msec 2024-11-21T21:54:36,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T21:54:36,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 410 msec 2024-11-21T21:54:36,239 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 2a884b411602,44867,1732226072579 had 2 regions 2024-11-21T21:54:36,240 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2a884b411602,44867,1732226072579, splitWal=true, meta=true, isMeta: false 2024-11-21T21:54:36,240 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44867,1732226072579-splitting dir is empty, no logs to split. 2024-11-21T21:54:36,240 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2a884b411602,44867,1732226072579 WAL count=0, meta=false 2024-11-21T21:54:36,241 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44867,1732226072579-splitting dir is empty, no logs to split. 2024-11-21T21:54:36,241 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2a884b411602,44867,1732226072579 WAL count=0, meta=false 2024-11-21T21:54:36,241 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 2a884b411602,44867,1732226072579 WAL splitting is done? wals=0, meta=false 2024-11-21T21:54:36,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732226073834, region=01696101048e3c1c33f4d7d02363e332, ASSIGN}] 2024-11-21T21:54:36,243 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732226073834, region=01696101048e3c1c33f4d7d02363e332, ASSIGN 2024-11-21T21:54:36,244 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732226073834, region=01696101048e3c1c33f4d7d02363e332, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T21:54:36,272 INFO [RS:2;2a884b411602:44029 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:54:36,272 INFO [RS:2;2a884b411602:44029 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2a884b411602%2C44029%2C1732226076049, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44029,1732226076049, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs, maxLogs=55 2024-11-21T21:54:36,292 INFO [RS:2;2a884b411602:44029 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/WALs/2a884b411602,44029,1732226076049/2a884b411602%2C44029%2C1732226076049.1732226076273 2024-11-21T21:54:36,293 DEBUG [RS:2;2a884b411602:44029 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T21:54:36,394 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(204): Hosts are {2a884b411602=0} racks are {/default-rack=0} 2024-11-21T21:54:36,395 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T21:54:36,395 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T21:54:36,395 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T21:54:36,395 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T21:54:36,395 INFO [2a884b411602:35339 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T21:54:36,395 INFO [2a884b411602:35339 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T21:54:36,396 DEBUG [2a884b411602:35339 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T21:54:36,396 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=01696101048e3c1c33f4d7d02363e332, regionState=OPENING, regionLocation=2a884b411602,40247,1732226072541 2024-11-21T21:54:36,399 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2a884b411602:44867 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:44867 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:54:36,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2a884b411602,44867,1732226072579, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2a884b411602,44867,1732226072579, seqNum=-1, error=java.net.ConnectException: Call to address=2a884b411602:44867 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:44867 2024-11-21T21:54:36,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2a884b411602,44867,1732226072579, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-21T21:54:36,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2a884b411602,44867,1732226072579, seqNum=-1 from cache 2024-11-21T21:54:36,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 2a884b411602:44867 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2a884b411602/172.17.0.2:44867 2024-11-21T21:54:36,498 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T21:54:36,498 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T21:54:36,500 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T21:54:36,500 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T21:54:36,500 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T21:54:36,500 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T21:54:36,514 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:54:36,515 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2a884b411602,40247,1732226072541, seqNum=-1] 2024-11-21T21:54:36,515 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:54:36,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44611, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:54:36,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732226073834, region=01696101048e3c1c33f4d7d02363e332, ASSIGN because future has completed 2024-11-21T21:54:36,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01696101048e3c1c33f4d7d02363e332, server=2a884b411602,40247,1732226072541}] 2024-11-21T21:54:36,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T21:54:36,669 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:54:36,669 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:54:36,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:36,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:36,670 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:54:36,670 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T21:54:36,670 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=567632812, stopped=false 2024-11-21T21:54:36,670 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2a884b411602,35339,1732226072389 2024-11-21T21:54:36,678 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:36,678 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 01696101048e3c1c33f4d7d02363e332, NAME => 'testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:54:36,679 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732226073834 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,679 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:54:36,680 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,680 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,681 INFO [StoreOpener-01696101048e3c1c33f4d7d02363e332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,683 INFO [StoreOpener-01696101048e3c1c33f4d7d02363e332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01696101048e3c1c33f4d7d02363e332 columnFamilyName family 2024-11-21T21:54:36,683 DEBUG [StoreOpener-01696101048e3c1c33f4d7d02363e332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:54:36,685 DEBUG [StoreFileOpener-01696101048e3c1c33f4d7d02363e332-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33, delay=901 ms 2024-11-21T21:54:36,687 DEBUG [StoreOpener-01696101048e3c1c33f4d7d02363e332-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33 2024-11-21T21:54:36,687 INFO [StoreOpener-01696101048e3c1c33f4d7d02363e332-1 {}] regionserver.HStore(327): Store=01696101048e3c1c33f4d7d02363e332/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:54:36,687 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,688 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,689 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,689 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,689 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,691 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,691 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 01696101048e3c1c33f4d7d02363e332; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74119708, jitterRate=0.10446971654891968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T21:54:36,691 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,692 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 01696101048e3c1c33f4d7d02363e332: Running coprocessor pre-open hook at 1732226076680Writing region info on filesystem at 1732226076680Initializing all the Stores at 1732226076681 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732226076681Cleaning up temporary data from old regions at 1732226076689 (+8 ms)Running coprocessor post-open hooks at 1732226076692 (+3 ms)Region opened successfully at 1732226076692 2024-11-21T21:54:36,693 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332., pid=11, masterSystemTime=1732226076673 2024-11-21T21:54:36,696 DEBUG [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:36,696 INFO [RS_OPEN_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:36,696 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=01696101048e3c1c33f4d7d02363e332, regionState=OPEN, openSeqNum=10, regionLocation=2a884b411602,40247,1732226072541 2024-11-21T21:54:36,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01696101048e3c1c33f4d7d02363e332, server=2a884b411602,40247,1732226072541 because future has completed 2024-11-21T21:54:36,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-21T21:54:36,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 01696101048e3c1c33f4d7d02363e332, server=2a884b411602,40247,1732226072541 in 179 msec 2024-11-21T21:54:36,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-21T21:54:36,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732226073834, region=01696101048e3c1c33f4d7d02363e332, ASSIGN in 460 msec 2024-11-21T21:54:36,704 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 2a884b411602,44867,1732226072579 after splitting done 2024-11-21T21:54:36,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 2a884b411602,44867,1732226072579, splitWal=true, meta=true in 892 msec 2024-11-21T21:54:36,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:36,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:36,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:36,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:36,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T21:54:36,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:36,739 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T21:54:36,739 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:54:36,740 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:54:36,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:36,740 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:36,741 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2a884b411602,40247,1732226072541' ***** 2024-11-21T21:54:36,741 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:36,741 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T21:54:36,741 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2a884b411602,44029,1732226076049' ***** 2024-11-21T21:54:36,741 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T21:54:36,741 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:54:36,741 INFO [RS:2;2a884b411602:44029 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T21:54:36,741 INFO [RS:0;2a884b411602:40247 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T21:54:36,742 INFO [RS:2;2a884b411602:44029 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.persistence 2024-11-21T21:54:36,742 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T21:54:36,742 INFO [RS:0;2a884b411602:40247 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a343941a-a4b7-8781-3373-4e6ef45b3236/bucket.persistence 2024-11-21T21:54:36,742 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T21:54:36,742 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T21:54:36,742 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T21:54:36,742 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T21:54:36,742 INFO [RS:2;2a884b411602:44029 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T21:54:36,742 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T21:54:36,742 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T21:54:36,743 INFO [RS:0;2a884b411602:40247 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T21:54:36,743 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T21:54:36,752 DEBUG [RS:2;2a884b411602:44029 {}] bucket.BucketCache(1487): Thread RS:2;2a884b411602:44029 started persisting bucket cache to file 2024-11-21T21:54:36,753 DEBUG [RS:2;2a884b411602:44029 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T21:54:36,753 DEBUG [RS:2;2a884b411602:44029 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T21:54:36,753 DEBUG [RS:0;2a884b411602:40247 {}] bucket.BucketCache(1487): Thread RS:0;2a884b411602:40247 started persisting bucket cache to file 2024-11-21T21:54:36,753 DEBUG [RS:0;2a884b411602:40247 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T21:54:36,753 DEBUG [RS:0;2a884b411602:40247 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T21:54:36,755 DEBUG [RS:0;2a884b411602:40247 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T21:54:36,755 DEBUG [RS:2;2a884b411602:44029 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T21:54:36,755 DEBUG [RS:2;2a884b411602:44029 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732226076753 2024-11-21T21:54:36,755 DEBUG [RS:0;2a884b411602:40247 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732226076753 2024-11-21T21:54:36,756 DEBUG [RS:0;2a884b411602:40247 {}] bucket.BucketCache(1509): Thread RS:0;2a884b411602:40247 finished persisting bucket cache to file, renaming 2024-11-21T21:54:36,756 INFO [RS:0;2a884b411602:40247 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T21:54:36,756 INFO [RS:0;2a884b411602:40247 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T21:54:36,756 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T21:54:36,756 INFO [RS:0;2a884b411602:40247 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T21:54:36,756 INFO [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(3091): Received CLOSE for 01696101048e3c1c33f4d7d02363e332 2024-11-21T21:54:36,756 INFO [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(959): stopping server 2a884b411602,40247,1732226072541 2024-11-21T21:54:36,756 INFO [RS:0;2a884b411602:40247 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:54:36,756 INFO [RS:0;2a884b411602:40247 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2a884b411602:40247. 2024-11-21T21:54:36,756 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T21:54:36,756 DEBUG [RS:0;2a884b411602:40247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:54:36,756 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 01696101048e3c1c33f4d7d02363e332, disabling compactions & flushes 2024-11-21T21:54:36,756 DEBUG [RS:0;2a884b411602:40247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:36,756 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:36,757 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:36,757 INFO [RS:0;2a884b411602:40247 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T21:54:36,757 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. after waiting 0 ms 2024-11-21T21:54:36,757 INFO [RS:0;2a884b411602:40247 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T21:54:36,757 INFO [RS:0;2a884b411602:40247 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T21:54:36,757 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:36,757 INFO [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T21:54:36,757 INFO [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T21:54:36,757 DEBUG [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(1325): Online Regions={01696101048e3c1c33f4d7d02363e332=testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332., 1588230740=hbase:meta,,1.1588230740} 2024-11-21T21:54:36,757 DEBUG [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(1351): Waiting on 01696101048e3c1c33f4d7d02363e332, 1588230740 2024-11-21T21:54:36,757 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T21:54:36,757 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T21:54:36,757 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T21:54:36,757 DEBUG [StoreFileCloser-01696101048e3c1c33f4d7d02363e332-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/family/8f9c33ae675f474cad5d225926761d33 2024-11-21T21:54:36,757 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T21:54:36,757 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T21:54:36,757 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T21:54:36,758 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-21T21:54:36,763 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/default/testBlockEvictionOnGracefulStop1732226073834/01696101048e3c1c33f4d7d02363e332/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-21T21:54:36,763 INFO [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:36,764 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 01696101048e3c1c33f4d7d02363e332: Waiting for close lock at 1732226076756Running coprocessor pre-close hooks at 1732226076756Disabling compacts and flushes for region at 1732226076756Disabling writes for close at 1732226076757 (+1 ms)Writing region close event to WAL at 1732226076758 (+1 ms)Running coprocessor post-close hooks at 1732226076763 (+5 ms)Closed at 1732226076763 2024-11-21T21:54:36,764 DEBUG [RS_CLOSE_REGION-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332. 2024-11-21T21:54:36,767 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6951b82ccb4948e1b06e0374a0b36970 2024-11-21T21:54:36,772 DEBUG [RS:2;2a884b411602:44029 {}] bucket.BucketCache(1509): Thread RS:2;2a884b411602:44029 finished persisting bucket cache to file, renaming 2024-11-21T21:54:36,772 WARN [RS:2;2a884b411602:44029 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-21T21:54:36,772 INFO [RS:2;2a884b411602:44029 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T21:54:36,773 INFO [RS:2;2a884b411602:44029 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T21:54:36,773 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T21:54:36,773 INFO [RS:2;2a884b411602:44029 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T21:54:36,773 INFO [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(959): stopping server 2a884b411602,44029,1732226076049 2024-11-21T21:54:36,773 INFO [RS:2;2a884b411602:44029 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:54:36,773 INFO [RS:2;2a884b411602:44029 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2a884b411602:44029. 2024-11-21T21:54:36,773 DEBUG [RS:2;2a884b411602:44029 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:54:36,773 DEBUG [RS:2;2a884b411602:44029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:36,773 INFO [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(976): stopping server 2a884b411602,44029,1732226076049; all regions closed. 2024-11-21T21:54:36,775 DEBUG [RS:2;2a884b411602:44029 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs 2024-11-21T21:54:36,775 INFO [RS:2;2a884b411602:44029 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2a884b411602%2C44029%2C1732226076049:(num 1732226076273) 2024-11-21T21:54:36,775 DEBUG [RS:2;2a884b411602:44029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:36,775 INFO [RS:2;2a884b411602:44029 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:54:36,775 INFO [RS:2;2a884b411602:44029 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:54:36,775 INFO [RS:2;2a884b411602:44029 {}] hbase.ChoreService(370): Chore service for: regionserver/2a884b411602:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T21:54:36,775 INFO [RS:2;2a884b411602:44029 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T21:54:36,775 INFO [regionserver/2a884b411602:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:54:36,775 INFO [RS:2;2a884b411602:44029 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T21:54:36,775 INFO [RS:2;2a884b411602:44029 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T21:54:36,776 INFO [RS:2;2a884b411602:44029 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:54:36,776 INFO [RS:2;2a884b411602:44029 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44029 2024-11-21T21:54:36,783 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/info/6951b82ccb4948e1b06e0374a0b36970 is 209, key is testBlockEvictionOnGracefulStop1732226073834,,1732226073838.01696101048e3c1c33f4d7d02363e332./info:regioninfo/1732226076696/Put/seqid=0 2024-11-21T21:54:36,783 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/info/6951b82ccb4948e1b06e0374a0b36970 2024-11-21T21:54:36,785 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/.tmp/info/6951b82ccb4948e1b06e0374a0b36970 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/6951b82ccb4948e1b06e0374a0b36970 2024-11-21T21:54:36,786 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/6951b82ccb4948e1b06e0374a0b36970, delay=902 ms 2024-11-21T21:54:36,787 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/6951b82ccb4948e1b06e0374a0b36970, entries=8, sequenceid=19, filesize=6.6 K 2024-11-21T21:54:36,788 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false 2024-11-21T21:54:36,789 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/2d4cefc0770546e4a6c2c8246b1025f6 2024-11-21T21:54:36,789 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/info/6951b82ccb4948e1b06e0374a0b36970 2024-11-21T21:54:36,790 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/ns/26573472c701433e84555f7c1e30558b 2024-11-21T21:54:36,791 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/table/e8a5e9c448374210b72bfd8d5183c840 2024-11-21T21:54:36,794 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-21T21:54:36,795 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T21:54:36,795 INFO [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T21:54:36,795 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732226076757Running coprocessor pre-close hooks at 1732226076757Disabling compacts and flushes for region at 1732226076757Disabling writes for close at 1732226076757Obtaining lock to block concurrent updates at 1732226076758 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732226076758Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732226076758Flushing stores of hbase:meta,,1.1588230740 at 1732226076758Flushing 1588230740/info: creating writer at 1732226076758Flushing 1588230740/info: appending metadata at 1732226076782 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732226076782Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d77449c: reopening flushed file at 1732226076785 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false at 1732226076789 (+4 ms)Writing region close event to WAL at 1732226076791 (+2 ms)Running coprocessor post-close hooks at 1732226076795 (+4 ms)Closed at 1732226076795 2024-11-21T21:54:36,795 DEBUG [RS_CLOSE_META-regionserver/2a884b411602:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T21:54:36,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:54:36,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2a884b411602,44029,1732226076049 2024-11-21T21:54:36,801 INFO [RS:2;2a884b411602:44029 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:54:36,814 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2a884b411602,44029,1732226076049] 2024-11-21T21:54:36,817 INFO [regionserver/2a884b411602:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:54:36,822 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2a884b411602,44029,1732226076049 already deleted, retry=false 2024-11-21T21:54:36,822 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2a884b411602,44029,1732226076049 expired; onlineServers=1 2024-11-21T21:54:36,839 INFO [regionserver/2a884b411602:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:54:36,914 INFO [RS:2;2a884b411602:44029 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:54:36,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:36,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44029-0x10160add1fb0003, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:36,914 INFO [RS:2;2a884b411602:44029 {}] regionserver.HRegionServer(1031): Exiting; stopping=2a884b411602,44029,1732226076049; zookeeper connection closed. 2024-11-21T21:54:36,914 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@37db8257 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@37db8257 2024-11-21T21:54:36,957 INFO [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(976): stopping server 2a884b411602,40247,1732226072541; all regions closed. 2024-11-21T21:54:36,959 DEBUG [RS:0;2a884b411602:40247 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs 2024-11-21T21:54:36,959 INFO [RS:0;2a884b411602:40247 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2a884b411602%2C40247%2C1732226072541.meta:.meta(num 1732226076181) 2024-11-21T21:54:36,961 DEBUG [RS:0;2a884b411602:40247 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/oldWALs 2024-11-21T21:54:36,961 INFO [RS:0;2a884b411602:40247 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2a884b411602%2C40247%2C1732226072541:(num 1732226073251) 2024-11-21T21:54:36,961 DEBUG [RS:0;2a884b411602:40247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:54:36,961 INFO [RS:0;2a884b411602:40247 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:54:36,961 INFO [RS:0;2a884b411602:40247 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:54:36,961 INFO [RS:0;2a884b411602:40247 {}] hbase.ChoreService(370): Chore service for: regionserver/2a884b411602:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T21:54:36,961 INFO [RS:0;2a884b411602:40247 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:54:36,961 INFO [regionserver/2a884b411602:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:54:36,961 INFO [RS:0;2a884b411602:40247 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40247 2024-11-21T21:54:36,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2a884b411602,40247,1732226072541 2024-11-21T21:54:36,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:54:36,980 INFO [RS:0;2a884b411602:40247 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:54:36,989 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2a884b411602,40247,1732226072541] 2024-11-21T21:54:36,997 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2a884b411602,40247,1732226072541 already deleted, retry=false 2024-11-21T21:54:36,997 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2a884b411602,40247,1732226072541 expired; onlineServers=0 2024-11-21T21:54:36,997 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2a884b411602,35339,1732226072389' ***** 2024-11-21T21:54:36,997 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T21:54:36,997 INFO [M:0;2a884b411602:35339 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:54:36,998 INFO [M:0;2a884b411602:35339 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:54:36,998 DEBUG [M:0;2a884b411602:35339 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T21:54:36,998 DEBUG [M:0;2a884b411602:35339 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T21:54:36,998 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T21:54:36,998 DEBUG [master/2a884b411602:0:becomeActiveMaster-HFileCleaner.large.0-1732226073027 {}] cleaner.HFileCleaner(306): Exit Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.large.0-1732226073027,5,FailOnTimeoutGroup] 2024-11-21T21:54:36,998 DEBUG [master/2a884b411602:0:becomeActiveMaster-HFileCleaner.small.0-1732226073027 {}] cleaner.HFileCleaner(306): Exit Thread[master/2a884b411602:0:becomeActiveMaster-HFileCleaner.small.0-1732226073027,5,FailOnTimeoutGroup] 2024-11-21T21:54:36,998 INFO [M:0;2a884b411602:35339 {}] hbase.ChoreService(370): Chore service for: master/2a884b411602:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T21:54:36,999 INFO [M:0;2a884b411602:35339 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:54:36,999 DEBUG [M:0;2a884b411602:35339 {}] master.HMaster(1795): Stopping service threads 2024-11-21T21:54:36,999 INFO [M:0;2a884b411602:35339 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T21:54:36,999 INFO [M:0;2a884b411602:35339 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T21:54:36,999 INFO [M:0;2a884b411602:35339 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T21:54:37,000 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T21:54:37,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T21:54:37,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:54:37,010 DEBUG [M:0;2a884b411602:35339 {}] zookeeper.ZKUtil(347): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T21:54:37,010 WARN [M:0;2a884b411602:35339 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T21:54:37,010 INFO [M:0;2a884b411602:35339 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/.lastflushedseqids 2024-11-21T21:54:37,016 INFO [M:0;2a884b411602:35339 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T21:54:37,016 INFO [M:0;2a884b411602:35339 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T21:54:37,016 DEBUG [M:0;2a884b411602:35339 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T21:54:37,016 INFO [M:0;2a884b411602:35339 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:37,016 DEBUG [M:0;2a884b411602:35339 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:37,016 DEBUG [M:0;2a884b411602:35339 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T21:54:37,016 DEBUG [M:0;2a884b411602:35339 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:37,016 INFO [M:0;2a884b411602:35339 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-21T21:54:37,027 DEBUG [M:0;2a884b411602:35339 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 435a4e4942b7453cb371778378173706 2024-11-21T21:54:37,044 DEBUG [M:0;2a884b411602:35339 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/435a4e4942b7453cb371778378173706 is 82, key is hbase:meta,,1/info:regioninfo/1732226076217/Put/seqid=0 2024-11-21T21:54:37,044 INFO [M:0;2a884b411602:35339 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/435a4e4942b7453cb371778378173706 2024-11-21T21:54:37,051 DEBUG [M:0;2a884b411602:35339 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a1a832224db64e45b76d2b92ef7bd3b6 2024-11-21T21:54:37,064 DEBUG [M:0;2a884b411602:35339 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a1a832224db64e45b76d2b92ef7bd3b6 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732226074222/Put/seqid=0 2024-11-21T21:54:37,065 INFO [M:0;2a884b411602:35339 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a1a832224db64e45b76d2b92ef7bd3b6 2024-11-21T21:54:37,072 DEBUG [M:0;2a884b411602:35339 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ddcf9f90e25e4c3284983a36a89b0b8f 2024-11-21T21:54:37,084 DEBUG [M:0;2a884b411602:35339 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ddcf9f90e25e4c3284983a36a89b0b8f is 69, key is 2a884b411602,40247,1732226072541/rs:state/1732226073053/Put/seqid=0 2024-11-21T21:54:37,085 INFO [M:0;2a884b411602:35339 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ddcf9f90e25e4c3284983a36a89b0b8f 2024-11-21T21:54:37,086 INFO [M:0;2a884b411602:35339 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ddcf9f90e25e4c3284983a36a89b0b8f 2024-11-21T21:54:37,087 DEBUG [M:0;2a884b411602:35339 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/435a4e4942b7453cb371778378173706 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/435a4e4942b7453cb371778378173706 2024-11-21T21:54:37,088 INFO [M:0;2a884b411602:35339 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/435a4e4942b7453cb371778378173706, entries=14, sequenceid=102, filesize=5.9 K 2024-11-21T21:54:37,089 DEBUG [M:0;2a884b411602:35339 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a1a832224db64e45b76d2b92ef7bd3b6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a1a832224db64e45b76d2b92ef7bd3b6 2024-11-21T21:54:37,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:37,089 INFO [RS:0;2a884b411602:40247 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:54:37,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40247-0x10160add1fb0001, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:37,089 INFO [RS:0;2a884b411602:40247 {}] regionserver.HRegionServer(1031): Exiting; stopping=2a884b411602,40247,1732226072541; zookeeper connection closed. 2024-11-21T21:54:37,089 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2c954ed2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2c954ed2 2024-11-21T21:54:37,089 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-21T21:54:37,090 INFO [M:0;2a884b411602:35339 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a1a832224db64e45b76d2b92ef7bd3b6, entries=11, sequenceid=102, filesize=6.5 K 2024-11-21T21:54:37,091 DEBUG [M:0;2a884b411602:35339 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ddcf9f90e25e4c3284983a36a89b0b8f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ddcf9f90e25e4c3284983a36a89b0b8f 2024-11-21T21:54:37,092 INFO [M:0;2a884b411602:35339 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ddcf9f90e25e4c3284983a36a89b0b8f 2024-11-21T21:54:37,092 INFO [M:0;2a884b411602:35339 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23a1e41a-7cd4-dd3d-8201-b9660287c9ff/13f009e4-9a88-bf7b-2354-b2ee1361fcb4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ddcf9f90e25e4c3284983a36a89b0b8f, entries=3, sequenceid=102, filesize=5.3 K 2024-11-21T21:54:37,093 INFO [M:0;2a884b411602:35339 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=102, compaction requested=false 2024-11-21T21:54:37,095 INFO [M:0;2a884b411602:35339 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:54:37,095 DEBUG [M:0;2a884b411602:35339 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732226077016Disabling compacts and flushes for region at 1732226077016Disabling writes for close at 1732226077016Obtaining lock to block concurrent updates at 1732226077016Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732226077016Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732226077017 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732226077017Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732226077017Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732226077043 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732226077043Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732226077046 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732226077064 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732226077064Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732226077066 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732226077083 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732226077083Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@621cca4b: reopening flushed file at 1732226077087 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5892e34: reopening flushed file at 1732226077088 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a7c3b89: reopening flushed file at 1732226077090 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=102, compaction requested=false at 1732226077093 (+3 ms)Writing region close event to WAL at 1732226077095 (+2 ms)Closed at 1732226077095 2024-11-21T21:54:37,096 INFO [M:0;2a884b411602:35339 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T21:54:37,096 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:54:37,096 INFO [M:0;2a884b411602:35339 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35339 2024-11-21T21:54:37,096 INFO [M:0;2a884b411602:35339 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:54:37,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:37,218 INFO [M:0;2a884b411602:35339 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:54:37,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35339-0x10160add1fb0000, quorum=127.0.0.1:53043, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:54:37,224 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T21:54:37,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T21:54:37,227 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:54:37,232 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 398), ProcessCount=11 (was 11), AvailableMemoryMB=4004 (was 4114)