2024-11-12 21:52:27,960 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-12 21:52:27,986 main DEBUG Took 0.023361 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 21:52:27,986 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 21:52:27,987 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 21:52:27,988 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 21:52:27,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:27,999 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 21:52:28,030 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,033 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,035 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,036 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,037 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,039 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,040 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,041 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,047 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,047 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,048 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,049 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,050 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,051 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,052 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,052 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,053 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,054 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,055 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,055 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 21:52:28,056 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,057 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 21:52:28,059 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 21:52:28,061 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 21:52:28,064 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 21:52:28,064 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 21:52:28,067 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 21:52:28,068 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 21:52:28,083 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 21:52:28,087 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 21:52:28,089 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 21:52:28,090 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 21:52:28,090 main DEBUG createAppenders(={Console}) 2024-11-12 21:52:28,091 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-12 21:52:28,092 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-12 21:52:28,092 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-12 21:52:28,093 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 21:52:28,093 main DEBUG OutputStream closed 2024-11-12 21:52:28,094 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 21:52:28,094 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 21:52:28,094 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-12 21:52:28,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 21:52:28,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 21:52:28,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 21:52:28,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 21:52:28,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 21:52:28,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 21:52:28,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 21:52:28,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 21:52:28,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 21:52:28,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 21:52:28,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 21:52:28,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 21:52:28,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 21:52:28,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 21:52:28,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 21:52:28,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 21:52:28,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 21:52:28,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 21:52:28,204 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 21:52:28,204 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-12 21:52:28,205 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 21:52:28,206 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-12T21:52:28,224 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-12 21:52:28,228 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 21:52:28,229 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T21:52:28,675 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9 2024-11-12T21:52:28,745 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=324, ProcessCount=11, AvailableMemoryMB=10191 2024-11-12T21:52:29,053 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T21:52:29,194 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:29,212 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/cluster_f16a91fa-f9e4-9612-b727-f37b48098ae0, deleteOnExit=true 2024-11-12T21:52:29,340 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/cluster_f16a91fa-f9e4-9612-b727-f37b48098ae0/zookeeper_0, clientPort=54704, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/cluster_f16a91fa-f9e4-9612-b727-f37b48098ae0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/cluster_f16a91fa-f9e4-9612-b727-f37b48098ae0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T21:52:29,359 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54704 2024-11-12T21:52:29,360 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:29,509 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a with version=8 2024-11-12T21:52:29,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/hbase-staging 2024-11-12T21:52:29,663 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T21:52:30,027 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0c576948786c:0 server-side Connection retries=45 2024-11-12T21:52:30,042 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:30,046 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:30,059 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T21:52:30,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:30,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T21:52:30,278 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T21:52:30,422 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T21:52:30,488 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T21:52:30,504 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T21:52:30,510 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T21:52:30,579 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 40390 (auto-detected) 2024-11-12T21:52:30,581 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T21:52:30,627 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45643 2024-11-12T21:52:30,665 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45643 connecting to ZooKeeper ensemble=127.0.0.1:54704 2024-11-12T21:52:30,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:456430x0, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T21:52:30,707 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45643-0x10035ec7eec0000 connected 2024-11-12T21:52:30,752 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:30,753 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:30,763 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:30,769 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a, hbase.cluster.distributed=false 2024-11-12T21:52:30,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T21:52:30,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45643 2024-11-12T21:52:30,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45643 2024-11-12T21:52:30,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45643 2024-11-12T21:52:30,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45643 2024-11-12T21:52:30,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45643 2024-11-12T21:52:30,920 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c576948786c:0 server-side Connection retries=45 2024-11-12T21:52:30,921 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:30,922 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:30,922 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T21:52:30,922 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:30,923 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T21:52:30,928 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T21:52:30,934 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T21:52:30,936 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37553 2024-11-12T21:52:30,939 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37553 connecting to ZooKeeper ensemble=127.0.0.1:54704 2024-11-12T21:52:30,940 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:30,941 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:30,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:375530x0, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T21:52:30,955 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:375530x0, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:30,964 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T21:52:30,969 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37553-0x10035ec7eec0001 connected 2024-11-12T21:52:30,981 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/bucket.cache 2024-11-12T21:52:30,987 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T21:52:30,988 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T21:52:30,990 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T21:52:30,993 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T21:52:30,993 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T21:52:31,000 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T21:52:31,004 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T21:52:31,004 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/bucket.persistence is complete. 2024-11-12T21:52:31,008 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T21:52:31,012 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T21:52:31,025 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T21:52:31,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37553 2024-11-12T21:52:31,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37553 2024-11-12T21:52:31,031 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37553 2024-11-12T21:52:31,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37553 2024-11-12T21:52:31,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37553 2024-11-12T21:52:31,061 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c576948786c:0 server-side Connection retries=45 2024-11-12T21:52:31,062 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:31,062 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:31,062 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T21:52:31,062 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:31,063 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T21:52:31,063 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T21:52:31,063 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T21:52:31,065 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33717 2024-11-12T21:52:31,069 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33717 connecting to ZooKeeper ensemble=127.0.0.1:54704 2024-11-12T21:52:31,070 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:31,071 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:31,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337170x0, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T21:52:31,081 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:337170x0, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:31,082 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T21:52:31,084 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33717-0x10035ec7eec0002 connected 2024-11-12T21:52:31,090 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/bucket.cache 2024-11-12T21:52:31,090 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T21:52:31,090 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T21:52:31,091 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T21:52:31,105 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T21:52:31,105 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T21:52:31,105 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T21:52:31,106 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T21:52:31,107 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/bucket.persistence is complete. 2024-11-12T21:52:31,117 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T21:52:31,119 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T21:52:31,123 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T21:52:31,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33717 2024-11-12T21:52:31,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33717 2024-11-12T21:52:31,191 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33717 2024-11-12T21:52:31,194 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33717 2024-11-12T21:52:31,194 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33717 2024-11-12T21:52:31,235 DEBUG [M:0;0c576948786c:45643 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0c576948786c:45643 2024-11-12T21:52:31,240 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0c576948786c,45643,1731448349780 2024-11-12T21:52:31,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:31,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:31,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:31,270 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0c576948786c,45643,1731448349780 2024-11-12T21:52:31,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T21:52:31,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:31,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T21:52:31,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:31,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:31,314 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T21:52:31,316 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0c576948786c,45643,1731448349780 from backup master directory 2024-11-12T21:52:31,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:31,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:31,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0c576948786c,45643,1731448349780 2024-11-12T21:52:31,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:31,324 WARN [master/0c576948786c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T21:52:31,324 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0c576948786c,45643,1731448349780 2024-11-12T21:52:31,327 INFO [master/0c576948786c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T21:52:31,331 INFO [master/0c576948786c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T21:52:31,365 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/hbase.id] with ID: 7f4311d5-4a8a-4ae7-9ac4-75541aaaaab5 2024-11-12T21:52:31,365 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/.tmp/hbase.id 2024-11-12T21:52:31,384 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/hbase.id] 2024-11-12T21:52:31,602 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:31,610 INFO [master/0c576948786c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T21:52:31,639 INFO [master/0c576948786c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-12T21:52:31,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:31,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:31,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:31,705 INFO [master/0c576948786c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T21:52:31,709 INFO [master/0c576948786c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T21:52:31,763 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T21:52:31,770 INFO [master/0c576948786c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T21:52:31,874 INFO [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store 2024-11-12T21:52:31,938 INFO [master/0c576948786c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T21:52:31,946 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:31,950 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T21:52:31,950 INFO [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:31,951 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:31,954 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T21:52:31,954 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:31,955 INFO [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:31,955 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731448351950Disabling compacts and flushes for region at 1731448351950Disabling writes for close at 1731448351954 (+4 ms)Writing region close event to WAL at 1731448351954Closed at 1731448351954 2024-11-12T21:52:31,956 WARN [master/0c576948786c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/.initializing 2024-11-12T21:52:31,956 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/WALs/0c576948786c,45643,1731448349780 2024-11-12T21:52:31,972 INFO [master/0c576948786c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T21:52:31,990 INFO [master/0c576948786c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c576948786c%2C45643%2C1731448349780, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/WALs/0c576948786c,45643,1731448349780, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/oldWALs, maxLogs=10 2024-11-12T21:52:32,045 INFO [master/0c576948786c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/WALs/0c576948786c,45643,1731448349780/0c576948786c%2C45643%2C1731448349780.1731448352000 2024-11-12T21:52:32,046 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T21:52:32,047 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T21:52:32,048 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:32,052 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,053 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,105 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T21:52:32,156 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:32,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:32,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,168 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T21:52:32,169 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:32,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:32,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T21:52:32,184 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:32,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:32,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T21:52:32,192 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:32,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:32,194 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,198 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,199 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,208 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,209 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,213 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T21:52:32,221 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:32,266 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T21:52:32,267 INFO [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62839712, jitterRate=-0.06361532211303711}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T21:52:32,279 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731448352070Initializing all the Stores at 1731448352071 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448352071Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448352072 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448352073 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448352073Cleaning up temporary data from old regions at 1731448352209 (+136 ms)Region opened successfully at 1731448352278 (+69 ms) 2024-11-12T21:52:32,285 INFO [master/0c576948786c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T21:52:32,354 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13aa50d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c576948786c/172.17.0.2:0 2024-11-12T21:52:32,391 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T21:52:32,406 INFO [master/0c576948786c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T21:52:32,407 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T21:52:32,411 INFO [master/0c576948786c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T21:52:32,412 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T21:52:32,419 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-12T21:52:32,420 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T21:52:32,451 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T21:52:32,461 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T21:52:32,463 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T21:52:32,467 INFO [master/0c576948786c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T21:52:32,472 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T21:52:32,474 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T21:52:32,477 INFO [master/0c576948786c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T21:52:32,483 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T21:52:32,485 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T21:52:32,488 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T21:52:32,490 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T21:52:32,515 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T21:52:32,517 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T21:52:32,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:32,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:32,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:32,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:32,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:32,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:32,544 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0c576948786c,45643,1731448349780, sessionid=0x10035ec7eec0000, setting cluster-up flag (Was=false) 2024-11-12T21:52:32,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:32,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:32,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:32,564 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T21:52:32,567 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c576948786c,45643,1731448349780 2024-11-12T21:52:32,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:32,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:32,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:32,577 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T21:52:32,579 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c576948786c,45643,1731448349780 2024-11-12T21:52:32,586 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T21:52:32,606 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(746): ClusterId : 7f4311d5-4a8a-4ae7-9ac4-75541aaaaab5 2024-11-12T21:52:32,606 INFO [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(746): ClusterId : 7f4311d5-4a8a-4ae7-9ac4-75541aaaaab5 2024-11-12T21:52:32,609 DEBUG [RS:0;0c576948786c:37553 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T21:52:32,609 DEBUG [RS:1;0c576948786c:33717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T21:52:32,614 DEBUG [RS:1;0c576948786c:33717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T21:52:32,614 DEBUG [RS:1;0c576948786c:33717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T21:52:32,615 DEBUG [RS:0;0c576948786c:37553 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T21:52:32,615 DEBUG [RS:0;0c576948786c:37553 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T21:52:32,619 DEBUG [RS:1;0c576948786c:33717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T21:52:32,619 DEBUG [RS:0;0c576948786c:37553 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T21:52:32,620 DEBUG [RS:0;0c576948786c:37553 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b74c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c576948786c/172.17.0.2:0 2024-11-12T21:52:32,620 DEBUG [RS:1;0c576948786c:33717 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@412b56e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c576948786c/172.17.0.2:0 2024-11-12T21:52:32,640 DEBUG [RS:0;0c576948786c:37553 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0c576948786c:37553 2024-11-12T21:52:32,645 INFO [RS:0;0c576948786c:37553 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T21:52:32,645 INFO [RS:0;0c576948786c:37553 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T21:52:32,645 DEBUG [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T21:52:32,650 DEBUG [RS:1;0c576948786c:33717 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0c576948786c:33717 2024-11-12T21:52:32,650 INFO [RS:1;0c576948786c:33717 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T21:52:32,650 INFO [RS:1;0c576948786c:33717 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T21:52:32,650 DEBUG [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T21:52:32,651 INFO [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c576948786c,45643,1731448349780 with port=37553, startcode=1731448350870 2024-11-12T21:52:32,652 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c576948786c,45643,1731448349780 with port=33717, startcode=1731448351061 2024-11-12T21:52:32,667 DEBUG [RS:1;0c576948786c:33717 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T21:52:32,667 DEBUG [RS:0;0c576948786c:37553 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T21:52:32,691 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T21:52:32,708 INFO [master/0c576948786c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T21:52:32,717 INFO [master/0c576948786c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T21:52:32,728 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0c576948786c,45643,1731448349780 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T21:52:32,743 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0c576948786c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T21:52:32,743 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0c576948786c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T21:52:32,743 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0c576948786c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T21:52:32,744 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0c576948786c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T21:52:32,744 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0c576948786c:0, corePoolSize=10, maxPoolSize=10 2024-11-12T21:52:32,744 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:32,744 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0c576948786c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T21:52:32,745 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:32,758 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T21:52:32,759 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T21:52:32,764 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:32,765 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T21:52:32,778 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731448382778 2024-11-12T21:52:32,779 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54255, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T21:52:32,780 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42311, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T21:52:32,781 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T21:52:32,784 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T21:52:32,788 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45643 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T21:52:32,790 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T21:52:32,790 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T21:52:32,791 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T21:52:32,791 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T21:52:32,795 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45643 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T21:52:32,810 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:32,818 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T21:52:32,819 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T21:52:32,820 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T21:52:32,823 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T21:52:32,824 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T21:52:32,825 DEBUG [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-12T21:52:32,825 DEBUG [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-12T21:52:32,825 WARN [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T21:52:32,825 WARN [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T21:52:32,829 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0c576948786c:0:becomeActiveMaster-HFileCleaner.large.0-1731448352825,5,FailOnTimeoutGroup] 2024-11-12T21:52:32,836 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0c576948786c:0:becomeActiveMaster-HFileCleaner.small.0-1731448352829,5,FailOnTimeoutGroup] 2024-11-12T21:52:32,836 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:32,837 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T21:52:32,838 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:32,839 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:32,846 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T21:52:32,847 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a 2024-11-12T21:52:32,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:32,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T21:52:32,868 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T21:52:32,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:32,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:32,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T21:52:32,877 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T21:52:32,877 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:32,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:32,878 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T21:52:32,883 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T21:52:32,883 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:32,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:32,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T21:52:32,889 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T21:52:32,890 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:32,890 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:32,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T21:52:32,892 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740 2024-11-12T21:52:32,892 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740 2024-11-12T21:52:32,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T21:52:32,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T21:52:32,894 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T21:52:32,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T21:52:32,909 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T21:52:32,909 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63415395, jitterRate=-0.05503697693347931}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T21:52:32,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731448352862Initializing all the Stores at 1731448352862Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448352863 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448352863Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448352863Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448352863Cleaning up temporary data from old regions at 1731448352893 (+30 ms)Region opened successfully at 1731448352915 (+22 ms) 2024-11-12T21:52:32,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T21:52:32,915 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T21:52:32,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T21:52:32,916 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T21:52:32,916 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T21:52:32,920 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T21:52:32,920 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731448352915Disabling compacts and flushes for region at 1731448352915Disabling writes for close at 1731448352916 (+1 ms)Writing region close event to WAL at 1731448352919 (+3 ms)Closed at 1731448352919 2024-11-12T21:52:32,926 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T21:52:32,926 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T21:52:32,927 INFO [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c576948786c,45643,1731448349780 with port=37553, startcode=1731448350870 2024-11-12T21:52:32,929 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c576948786c,45643,1731448349780 with port=33717, startcode=1731448351061 2024-11-12T21:52:32,930 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45643 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c576948786c,37553,1731448350870 2024-11-12T21:52:32,934 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45643 {}] master.ServerManager(517): Registering regionserver=0c576948786c,37553,1731448350870 2024-11-12T21:52:32,946 DEBUG [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a 2024-11-12T21:52:32,947 DEBUG [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T21:52:32,947 DEBUG [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T21:52:32,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T21:52:32,951 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45643 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c576948786c,33717,1731448351061 2024-11-12T21:52:32,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T21:52:32,952 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45643 {}] master.ServerManager(517): Registering regionserver=0c576948786c,33717,1731448351061 2024-11-12T21:52:32,953 DEBUG [RS:0;0c576948786c:37553 {}] zookeeper.ZKUtil(111): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c576948786c,37553,1731448350870 2024-11-12T21:52:32,953 WARN [RS:0;0c576948786c:37553 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T21:52:32,954 INFO [RS:0;0c576948786c:37553 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T21:52:32,954 DEBUG [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,37553,1731448350870 2024-11-12T21:52:32,959 DEBUG [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a 2024-11-12T21:52:32,959 DEBUG [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T21:52:32,959 DEBUG [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T21:52:32,965 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T21:52:32,966 DEBUG [RS:1;0c576948786c:33717 {}] zookeeper.ZKUtil(111): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c576948786c,33717,1731448351061 2024-11-12T21:52:32,966 WARN [RS:1;0c576948786c:33717 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T21:52:32,967 INFO [RS:1;0c576948786c:33717 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T21:52:32,967 DEBUG [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33717,1731448351061 2024-11-12T21:52:32,968 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T21:52:32,980 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c576948786c,33717,1731448351061] 2024-11-12T21:52:32,980 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c576948786c,37553,1731448350870] 2024-11-12T21:52:33,010 INFO [RS:0;0c576948786c:37553 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T21:52:33,010 INFO [RS:1;0c576948786c:33717 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T21:52:33,034 INFO [RS:1;0c576948786c:33717 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T21:52:33,037 INFO [RS:0;0c576948786c:37553 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T21:52:33,044 INFO [RS:1;0c576948786c:33717 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T21:52:33,044 INFO [RS:0;0c576948786c:37553 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T21:52:33,044 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,044 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,046 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T21:52:33,049 INFO [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T21:52:33,052 INFO [RS:0;0c576948786c:37553 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T21:52:33,052 INFO [RS:1;0c576948786c:33717 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T21:52:33,054 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,054 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,054 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,054 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,054 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,054 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,054 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,055 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,055 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,055 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,055 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,055 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,055 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c576948786c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T21:52:33,055 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c576948786c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T21:52:33,055 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,056 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,056 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,056 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,056 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,056 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,056 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,056 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,056 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,056 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,056 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,057 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:33,057 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c576948786c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T21:52:33,057 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c576948786c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T21:52:33,057 DEBUG [RS:1;0c576948786c:33717 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c576948786c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T21:52:33,057 DEBUG [RS:0;0c576948786c:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c576948786c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T21:52:33,063 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,063 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,063 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,063 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,063 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,063 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,064 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,064 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,064 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,064 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,064 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,37553,1731448350870-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T21:52:33,064 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,33717,1731448351061-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T21:52:33,088 INFO [RS:0;0c576948786c:37553 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T21:52:33,090 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,37553,1731448350870-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,090 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,091 INFO [RS:0;0c576948786c:37553 {}] regionserver.Replication(171): 0c576948786c,37553,1731448350870 started 2024-11-12T21:52:33,093 INFO [RS:1;0c576948786c:33717 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T21:52:33,093 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,33717,1731448351061-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,093 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,094 INFO [RS:1;0c576948786c:33717 {}] regionserver.Replication(171): 0c576948786c,33717,1731448351061 started 2024-11-12T21:52:33,113 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,113 INFO [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(1482): Serving as 0c576948786c,37553,1731448350870, RpcServer on 0c576948786c/172.17.0.2:37553, sessionid=0x10035ec7eec0001 2024-11-12T21:52:33,114 DEBUG [RS:0;0c576948786c:37553 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T21:52:33,114 DEBUG [RS:0;0c576948786c:37553 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c576948786c,37553,1731448350870 2024-11-12T21:52:33,114 DEBUG [RS:0;0c576948786c:37553 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c576948786c,37553,1731448350870' 2024-11-12T21:52:33,115 DEBUG [RS:0;0c576948786c:37553 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T21:52:33,116 DEBUG [RS:0;0c576948786c:37553 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T21:52:33,116 DEBUG [RS:0;0c576948786c:37553 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T21:52:33,116 DEBUG [RS:0;0c576948786c:37553 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T21:52:33,117 DEBUG [RS:0;0c576948786c:37553 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c576948786c,37553,1731448350870 2024-11-12T21:52:33,117 DEBUG [RS:0;0c576948786c:37553 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c576948786c,37553,1731448350870' 2024-11-12T21:52:33,117 DEBUG [RS:0;0c576948786c:37553 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T21:52:33,117 DEBUG [RS:0;0c576948786c:37553 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T21:52:33,117 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,118 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(1482): Serving as 0c576948786c,33717,1731448351061, RpcServer on 0c576948786c/172.17.0.2:33717, sessionid=0x10035ec7eec0002 2024-11-12T21:52:33,118 DEBUG [RS:1;0c576948786c:33717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T21:52:33,118 DEBUG [RS:0;0c576948786c:37553 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T21:52:33,118 DEBUG [RS:1;0c576948786c:33717 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c576948786c,33717,1731448351061 2024-11-12T21:52:33,118 DEBUG [RS:1;0c576948786c:33717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c576948786c,33717,1731448351061' 2024-11-12T21:52:33,118 INFO [RS:0;0c576948786c:37553 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T21:52:33,118 DEBUG [RS:1;0c576948786c:33717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T21:52:33,118 INFO [RS:0;0c576948786c:37553 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T21:52:33,119 DEBUG [RS:1;0c576948786c:33717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T21:52:33,120 WARN [0c576948786c:45643 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T21:52:33,121 DEBUG [RS:1;0c576948786c:33717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T21:52:33,121 DEBUG [RS:1;0c576948786c:33717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T21:52:33,121 DEBUG [RS:1;0c576948786c:33717 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c576948786c,33717,1731448351061 2024-11-12T21:52:33,121 DEBUG [RS:1;0c576948786c:33717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c576948786c,33717,1731448351061' 2024-11-12T21:52:33,121 DEBUG [RS:1;0c576948786c:33717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T21:52:33,122 DEBUG [RS:1;0c576948786c:33717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T21:52:33,123 DEBUG [RS:1;0c576948786c:33717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T21:52:33,123 INFO [RS:1;0c576948786c:33717 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T21:52:33,123 INFO [RS:1;0c576948786c:33717 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T21:52:33,225 INFO [RS:1;0c576948786c:33717 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T21:52:33,225 INFO [RS:0;0c576948786c:37553 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T21:52:33,226 INFO [RS:1;0c576948786c:33717 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0c576948786c%2C33717%2C1731448351061, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33717,1731448351061, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs, maxLogs=55 2024-11-12T21:52:33,227 INFO [RS:0;0c576948786c:37553 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0c576948786c%2C37553%2C1731448350870, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,37553,1731448350870, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs, maxLogs=55 2024-11-12T21:52:33,250 INFO [RS:0;0c576948786c:37553 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,37553,1731448350870/0c576948786c%2C37553%2C1731448350870.1731448353231 2024-11-12T21:52:33,250 INFO [RS:1;0c576948786c:33717 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33717,1731448351061/0c576948786c%2C33717%2C1731448351061.1731448353230 2024-11-12T21:52:33,250 DEBUG [RS:0;0c576948786c:37553 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T21:52:33,250 DEBUG [RS:1;0c576948786c:33717 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T21:52:33,372 DEBUG [0c576948786c:45643 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-12T21:52:33,383 DEBUG [0c576948786c:45643 {}] balancer.BalancerClusterState(204): Hosts are {0c576948786c=0} racks are {/default-rack=0} 2024-11-12T21:52:33,386 DEBUG [0c576948786c:45643 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T21:52:33,386 DEBUG [0c576948786c:45643 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T21:52:33,386 DEBUG [0c576948786c:45643 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T21:52:33,386 DEBUG [0c576948786c:45643 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T21:52:33,386 INFO [0c576948786c:45643 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T21:52:33,386 INFO [0c576948786c:45643 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T21:52:33,386 DEBUG [0c576948786c:45643 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T21:52:33,393 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0c576948786c,33717,1731448351061 2024-11-12T21:52:33,398 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c576948786c,33717,1731448351061, state=OPENING 2024-11-12T21:52:33,402 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T21:52:33,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:33,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:33,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:33,405 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:33,405 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:33,405 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:33,408 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T21:52:33,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0c576948786c,33717,1731448351061}] 2024-11-12T21:52:33,586 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T21:52:33,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34893, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T21:52:33,600 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T21:52:33,600 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T21:52:33,601 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T21:52:33,603 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0c576948786c%2C33717%2C1731448351061.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33717,1731448351061, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs, maxLogs=55 2024-11-12T21:52:33,626 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33717,1731448351061/0c576948786c%2C33717%2C1731448351061.meta.1731448353605.meta 2024-11-12T21:52:33,627 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T21:52:33,628 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T21:52:33,629 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T21:52:33,632 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T21:52:33,637 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T21:52:33,641 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T21:52:33,642 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:33,642 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T21:52:33,642 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T21:52:33,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T21:52:33,646 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T21:52:33,646 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:33,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:33,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T21:52:33,647 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T21:52:33,648 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:33,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:33,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T21:52:33,649 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T21:52:33,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:33,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:33,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T21:52:33,650 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T21:52:33,650 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:33,651 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:33,651 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T21:52:33,652 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740 2024-11-12T21:52:33,653 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740 2024-11-12T21:52:33,654 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T21:52:33,654 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T21:52:33,654 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T21:52:33,656 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T21:52:33,656 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59580368, jitterRate=-0.1121833324432373}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T21:52:33,657 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T21:52:33,658 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731448353643Writing region info on filesystem at 1731448353643Initializing all the Stores at 1731448353644 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448353644Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448353644Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448353644Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448353644Cleaning up temporary data from old regions at 1731448353654 (+10 ms)Running coprocessor post-open hooks at 1731448353657 (+3 ms)Region opened successfully at 1731448353658 (+1 ms) 2024-11-12T21:52:33,664 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731448353577 2024-11-12T21:52:33,676 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T21:52:33,676 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T21:52:33,677 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0c576948786c,33717,1731448351061 2024-11-12T21:52:33,679 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c576948786c,33717,1731448351061, state=OPEN 2024-11-12T21:52:33,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:33,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:33,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:33,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:33,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:33,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:33,682 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0c576948786c,33717,1731448351061 2024-11-12T21:52:33,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T21:52:33,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0c576948786c,33717,1731448351061 in 272 msec 2024-11-12T21:52:33,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T21:52:33,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 741 msec 2024-11-12T21:52:33,697 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T21:52:33,697 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T21:52:33,719 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T21:52:33,720 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c576948786c,33717,1731448351061, seqNum=-1] 2024-11-12T21:52:33,742 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T21:52:33,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52259, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T21:52:33,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1340 sec 2024-11-12T21:52:33,766 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731448353766, completionTime=-1 2024-11-12T21:52:33,770 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-12T21:52:33,770 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T21:52:33,808 INFO [master/0c576948786c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T21:52:33,808 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731448413808 2024-11-12T21:52:33,808 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731448473808 2024-11-12T21:52:33,808 INFO [master/0c576948786c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 38 msec 2024-11-12T21:52:33,811 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-12T21:52:33,819 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,45643,1731448349780-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,820 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,45643,1731448349780-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,820 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,45643,1731448349780-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,822 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0c576948786c:45643, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,823 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,823 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,830 DEBUG [master/0c576948786c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T21:52:33,856 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.531sec 2024-11-12T21:52:33,858 INFO [master/0c576948786c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T21:52:33,860 INFO [master/0c576948786c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T21:52:33,861 INFO [master/0c576948786c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T21:52:33,862 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T21:52:33,862 INFO [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T21:52:33,863 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,45643,1731448349780-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T21:52:33,863 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,45643,1731448349780-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T21:52:33,865 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T21:52:33,866 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T21:52:33,866 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,45643,1731448349780-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:33,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e2a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T21:52:33,929 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T21:52:33,929 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T21:52:33,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0c576948786c,45643,-1 for getting cluster id 2024-11-12T21:52:33,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T21:52:33,946 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f4311d5-4a8a-4ae7-9ac4-75541aaaaab5' 2024-11-12T21:52:33,949 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T21:52:33,949 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f4311d5-4a8a-4ae7-9ac4-75541aaaaab5" 2024-11-12T21:52:33,950 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376d0125, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T21:52:33,950 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0c576948786c,45643,-1] 2024-11-12T21:52:33,954 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T21:52:33,957 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:33,957 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T21:52:33,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d954e30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T21:52:33,961 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T21:52:33,967 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c576948786c,33717,1731448351061, seqNum=-1] 2024-11-12T21:52:33,968 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T21:52:33,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T21:52:33,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0c576948786c,45643,1731448349780 2024-11-12T21:52:34,004 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T21:52:34,009 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0c576948786c,45643,1731448349780 2024-11-12T21:52:34,014 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2cc653a1 2024-11-12T21:52:34,015 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T21:52:34,018 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T21:52:34,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731448353999', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T21:52:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731448353999 2024-11-12T21:52:34,032 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731448353999 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T21:52:34,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731448353999" procId is: 4 2024-11-12T21:52:34,034 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:34,035 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731448353999 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T21:52:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T21:52:34,046 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731448353999-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a37659093bf81c8099642b42be820b91, NAME => 'testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731448353999', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a 2024-11-12T21:52:34,051 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731448353999-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:34,051 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731448353999-pool-0 {}] regionserver.HRegion(1722): Closing a37659093bf81c8099642b42be820b91, disabling compactions & flushes 2024-11-12T21:52:34,051 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731448353999-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:34,052 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731448353999-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:34,052 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731448353999-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. after waiting 0 ms 2024-11-12T21:52:34,052 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731448353999-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:34,052 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731448353999-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:34,052 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731448353999-pool-0 {}] regionserver.HRegion(1676): Region close journal for a37659093bf81c8099642b42be820b91: Waiting for close lock at 1731448354051Disabling compacts and flushes for region at 1731448354051Disabling writes for close at 1731448354052 (+1 ms)Writing region close event to WAL at 1731448354052Closed at 1731448354052 2024-11-12T21:52:34,054 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731448353999 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T21:52:34,059 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731448354054"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731448354054"}]},"ts":"1731448354054"} 2024-11-12T21:52:34,064 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T21:52:34,065 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731448353999 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T21:52:34,068 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731448353999","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731448354065"}]},"ts":"1731448354065"} 2024-11-12T21:52:34,072 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731448353999, state=ENABLING in hbase:meta 2024-11-12T21:52:34,072 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0c576948786c=0} racks are {/default-rack=0} 2024-11-12T21:52:34,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T21:52:34,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T21:52:34,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T21:52:34,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T21:52:34,073 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T21:52:34,073 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T21:52:34,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T21:52:34,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, ASSIGN}] 2024-11-12T21:52:34,076 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, ASSIGN 2024-11-12T21:52:34,076 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, ASSIGN; state=OFFLINE, location=0c576948786c,37553,1731448350870; forceNewPlan=false, retain=false 2024-11-12T21:52:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T21:52:34,232 INFO [0c576948786c:45643 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T21:52:34,234 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a37659093bf81c8099642b42be820b91, regionState=OPENING, regionLocation=0c576948786c,37553,1731448350870 2024-11-12T21:52:34,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, ASSIGN because future has completed 2024-11-12T21:52:34,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a37659093bf81c8099642b42be820b91, server=0c576948786c,37553,1731448350870}] 2024-11-12T21:52:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T21:52:34,392 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T21:52:34,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43415, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T21:52:34,399 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:34,399 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a37659093bf81c8099642b42be820b91, NAME => 'testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91.', STARTKEY => '', ENDKEY => ''} 2024-11-12T21:52:34,400 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731448353999 a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,400 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:34,400 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,400 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,402 INFO [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,407 INFO [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a37659093bf81c8099642b42be820b91 columnFamilyName family 2024-11-12T21:52:34,407 DEBUG [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:34,408 INFO [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] regionserver.HStore(327): Store=a37659093bf81c8099642b42be820b91/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:34,408 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,409 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,409 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,409 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,409 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,410 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,416 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T21:52:34,416 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a37659093bf81c8099642b42be820b91; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63130070, jitterRate=-0.05928865075111389}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T21:52:34,416 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:34,417 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a37659093bf81c8099642b42be820b91: Running coprocessor pre-open hook at 1731448354401Writing region info on filesystem at 1731448354401Initializing all the Stores at 1731448354401Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448354401Cleaning up temporary data from old regions at 1731448354409 (+8 ms)Running coprocessor post-open hooks at 1731448354416 (+7 ms)Region opened successfully at 1731448354417 (+1 ms) 2024-11-12T21:52:34,418 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91., pid=6, masterSystemTime=1731448354392 2024-11-12T21:52:34,421 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:34,421 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:34,422 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a37659093bf81c8099642b42be820b91, regionState=OPEN, openSeqNum=2, regionLocation=0c576948786c,37553,1731448350870 2024-11-12T21:52:34,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a37659093bf81c8099642b42be820b91, server=0c576948786c,37553,1731448350870 because future has completed 2024-11-12T21:52:34,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T21:52:34,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a37659093bf81c8099642b42be820b91, server=0c576948786c,37553,1731448350870 in 188 msec 2024-11-12T21:52:34,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T21:52:34,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, ASSIGN in 355 msec 2024-11-12T21:52:34,432 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731448353999 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T21:52:34,432 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731448353999","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731448354432"}]},"ts":"1731448354432"} 2024-11-12T21:52:34,435 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731448353999, state=ENABLED in hbase:meta 2024-11-12T21:52:34,436 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731448353999 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T21:52:34,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731448353999 in 409 msec 2024-11-12T21:52:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T21:52:34,666 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731448353999 get assigned. Timeout = 60000ms 2024-11-12T21:52:34,666 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731448353999 completed 2024-11-12T21:52:34,669 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T21:52:34,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731448353999 assigned to meta. Checking AM states. 2024-11-12T21:52:34,675 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T21:52:34,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731448353999 assigned. 2024-11-12T21:52:34,684 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731448353999', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91., hostname=0c576948786c,37553,1731448350870, seqNum=2] 2024-11-12T21:52:34,685 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T21:52:34,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T21:52:34,698 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a37659093bf81c8099642b42be820b91 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-12T21:52:34,722 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1f5f4a985c3945308daf60635ab38b6a 2024-11-12T21:52:34,761 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/.tmp/family/1f5f4a985c3945308daf60635ab38b6a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-12T21:52:34,766 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/.tmp/family/1f5f4a985c3945308daf60635ab38b6a 2024-11-12T21:52:34,799 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/.tmp/family/1f5f4a985c3945308daf60635ab38b6a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a 2024-11-12T21:52:34,800 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a, delay=949 ms 2024-11-12T21:52:34,802 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a, entries=2, sequenceid=6, filesize=4.9 K 2024-11-12T21:52:34,807 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for a37659093bf81c8099642b42be820b91 in 109ms, sequenceid=6, compaction requested=false 2024-11-12T21:52:34,808 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731448353999' 2024-11-12T21:52:34,810 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a37659093bf81c8099642b42be820b91: 2024-11-12T21:52:35,752 DEBUG [hfile-prefetch-1731448354800 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1f5f4a985c3945308daf60635ab38b6a_0. 2024-11-12T21:52:35,757 DEBUG [hfile-prefetch-1731448354800 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1f5f4a985c3945308daf60635ab38b6a_117. 2024-11-12T21:52:35,758 DEBUG [hfile-prefetch-1731448354800 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a, with total blocks 2, and data blocks 1 2024-11-12T21:52:35,758 DEBUG [hfile-prefetch-1731448354800 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-12T21:52:35,759 DEBUG [hfile-prefetch-1731448354800 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-12T21:52:35,759 DEBUG [hfile-prefetch-1731448354800 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a. Will sleep 100ms and try the verification again. 2024-11-12T21:52:35,798 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T21:52:35,798 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T21:52:35,799 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T21:52:35,799 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T21:52:35,803 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T21:52:35,803 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T21:52:35,803 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T21:52:35,804 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T21:52:35,812 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T21:52:35,812 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731448355803 2024-11-12T21:52:35,812 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T21:52:35,815 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-12T21:52:35,815 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731448355798 2024-11-12T21:52:35,815 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T21:52:35,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=a37659093bf81c8099642b42be820b91, source=0c576948786c,37553,1731448350870, destination=0c576948786c,33717,1731448351061, warming up region on 0c576948786c,33717,1731448351061 2024-11-12T21:52:35,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=a37659093bf81c8099642b42be820b91, source=0c576948786c,37553,1731448350870, destination=0c576948786c,33717,1731448351061, running balancer 2024-11-12T21:52:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, REOPEN/MOVE 2024-11-12T21:52:35,823 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, REOPEN/MOVE 2024-11-12T21:52:35,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] regionserver.HRegion(7855): Warmup {ENCODED => a37659093bf81c8099642b42be820b91, NAME => 'testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91.', STARTKEY => '', ENDKEY => ''} 2024-11-12T21:52:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:35,824 INFO [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a37659093bf81c8099642b42be820b91 2024-11-12T21:52:35,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a37659093bf81c8099642b42be820b91, regionState=CLOSING, regionLocation=0c576948786c,37553,1731448350870 2024-11-12T21:52:35,825 INFO [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a37659093bf81c8099642b42be820b91 columnFamilyName family 2024-11-12T21:52:35,825 DEBUG [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:35,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, REOPEN/MOVE because future has completed 2024-11-12T21:52:35,828 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-12T21:52:35,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure a37659093bf81c8099642b42be820b91, server=0c576948786c,37553,1731448350870}] 2024-11-12T21:52:35,829 DEBUG [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a 2024-11-12T21:52:35,829 INFO [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] regionserver.HStore(327): Store=a37659093bf81c8099642b42be820b91/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] regionserver.HRegion(1722): Closing a37659093bf81c8099642b42be820b91, disabling compactions & flushes 2024-11-12T21:52:35,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. after waiting 0 ms 2024-11-12T21:52:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:35,830 DEBUG [StoreFileCloser-a37659093bf81c8099642b42be820b91-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a 2024-11-12T21:52:35,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33717 {}] regionserver.HRegion(1676): Region close journal for a37659093bf81c8099642b42be820b91: Waiting for close lock at 1731448355829Disabling compacts and flushes for region at 1731448355829Disabling writes for close at 1731448355829Writing region close event to WAL at 1731448355831 (+2 ms)Closed at 1731448355831 2024-11-12T21:52:35,833 WARN [hfile-prefetch-1731448354800 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-12T21:52:35,834 DEBUG [hfile-prefetch-1731448354800 {}] hfile.PrefetchExecutor(121): Prefetch completed for 1f5f4a985c3945308daf60635ab38b6a 2024-11-12T21:52:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-12T21:52:35,991 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close a37659093bf81c8099642b42be820b91 2024-11-12T21:52:35,991 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-12T21:52:35,992 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing a37659093bf81c8099642b42be820b91, disabling compactions & flushes 2024-11-12T21:52:35,992 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:35,992 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:35,992 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. after waiting 0 ms 2024-11-12T21:52:35,993 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:35,994 DEBUG [StoreFileCloser-a37659093bf81c8099642b42be820b91-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 1f5f4a985c3945308daf60635ab38b6a, starting offset: 0, end offset: 9223372036854775807 2024-11-12T21:52:35,996 DEBUG [StoreFileCloser-a37659093bf81c8099642b42be820b91-family-1 {}] bucket.BucketCache(857): removed key 1f5f4a985c3945308daf60635ab38b6a_0 from back map with offset lock 2094080 in the evict process 2024-11-12T21:52:36,001 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-12T21:52:36,002 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:36,002 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for a37659093bf81c8099642b42be820b91: Waiting for close lock at 1731448355992Running coprocessor pre-close hooks at 1731448355992Disabling compacts and flushes for region at 1731448355992Disabling writes for close at 1731448355992Writing region close event to WAL at 1731448355997 (+5 ms)Running coprocessor post-close hooks at 1731448356001 (+4 ms)Closed at 1731448356002 (+1 ms) 2024-11-12T21:52:36,003 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding a37659093bf81c8099642b42be820b91 move to 0c576948786c,33717,1731448351061 record at close sequenceid=6 2024-11-12T21:52:36,006 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,006 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a37659093bf81c8099642b42be820b91, regionState=CLOSED 2024-11-12T21:52:36,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure a37659093bf81c8099642b42be820b91, server=0c576948786c,37553,1731448350870 because future has completed 2024-11-12T21:52:36,012 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T21:52:36,012 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T21:52:36,012 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T21:52:36,012 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T21:52:36,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T21:52:36,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure a37659093bf81c8099642b42be820b91, server=0c576948786c,37553,1731448350870 in 182 msec 2024-11-12T21:52:36,014 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, REOPEN/MOVE; state=CLOSED, location=0c576948786c,33717,1731448351061; forceNewPlan=false, retain=false 2024-11-12T21:52:36,015 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T21:52:36,015 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731448356012 2024-11-12T21:52:36,015 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T21:52:36,164 INFO [0c576948786c:45643 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T21:52:36,165 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a37659093bf81c8099642b42be820b91, regionState=OPENING, regionLocation=0c576948786c,33717,1731448351061 2024-11-12T21:52:36,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, REOPEN/MOVE because future has completed 2024-11-12T21:52:36,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a37659093bf81c8099642b42be820b91, server=0c576948786c,33717,1731448351061}] 2024-11-12T21:52:36,330 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:36,330 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => a37659093bf81c8099642b42be820b91, NAME => 'testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91.', STARTKEY => '', ENDKEY => ''} 2024-11-12T21:52:36,331 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731448353999 a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,331 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:36,332 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,332 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,333 INFO [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,335 INFO [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a37659093bf81c8099642b42be820b91 columnFamilyName family 2024-11-12T21:52:36,335 DEBUG [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:36,338 DEBUG [StoreFileOpener-a37659093bf81c8099642b42be820b91-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a, delay=924 ms 2024-11-12T21:52:36,340 DEBUG [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a 2024-11-12T21:52:36,341 INFO [StoreOpener-a37659093bf81c8099642b42be820b91-1 {}] regionserver.HStore(327): Store=a37659093bf81c8099642b42be820b91/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:36,341 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,342 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,342 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,343 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,343 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,347 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,347 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened a37659093bf81c8099642b42be820b91; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66080236, jitterRate=-0.015327751636505127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T21:52:36,347 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,348 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for a37659093bf81c8099642b42be820b91: Running coprocessor pre-open hook at 1731448356332Writing region info on filesystem at 1731448356332Initializing all the Stores at 1731448356333 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448356333Cleaning up temporary data from old regions at 1731448356343 (+10 ms)Running coprocessor post-open hooks at 1731448356347 (+4 ms)Region opened successfully at 1731448356348 (+1 ms) 2024-11-12T21:52:36,349 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91., pid=9, masterSystemTime=1731448356327 2024-11-12T21:52:36,352 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:36,352 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:36,352 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a37659093bf81c8099642b42be820b91, regionState=OPEN, openSeqNum=10, regionLocation=0c576948786c,33717,1731448351061 2024-11-12T21:52:36,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a37659093bf81c8099642b42be820b91, server=0c576948786c,33717,1731448351061 because future has completed 2024-11-12T21:52:36,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-12T21:52:36,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure a37659093bf81c8099642b42be820b91, server=0c576948786c,33717,1731448351061 in 182 msec 2024-11-12T21:52:36,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731448353999, region=a37659093bf81c8099642b42be820b91, REOPEN/MOVE in 538 msec 2024-11-12T21:52:36,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T21:52:36,376 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T21:52:36,376 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T21:52:36,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:36,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:36,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T21:52:36,383 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T21:52:36,383 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1596482003, stopped=false 2024-11-12T21:52:36,383 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0c576948786c,45643,1731448349780 2024-11-12T21:52:36,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:36,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:36,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:36,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:36,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:36,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:36,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T21:52:36,386 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:36,386 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:36,386 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T21:52:36,386 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:36,386 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T21:52:36,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:36,387 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c576948786c,37553,1731448350870' ***** 2024-11-12T21:52:36,387 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T21:52:36,387 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c576948786c,33717,1731448351061' ***** 2024-11-12T21:52:36,387 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T21:52:36,387 INFO [RS:0;0c576948786c:37553 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T21:52:36,387 INFO [RS:1;0c576948786c:33717 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T21:52:36,388 INFO [RS:1;0c576948786c:33717 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/bucket.persistence 2024-11-12T21:52:36,388 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T21:52:36,388 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T21:52:36,388 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T21:52:36,388 INFO [RS:0;0c576948786c:37553 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/bucket.persistence 2024-11-12T21:52:36,388 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T21:52:36,388 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T21:52:36,388 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T21:52:36,388 INFO [RS:0;0c576948786c:37553 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T21:52:36,388 INFO [RS:1;0c576948786c:33717 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T21:52:36,388 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T21:52:36,388 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T21:52:36,398 DEBUG [RS:0;0c576948786c:37553 {}] bucket.BucketCache(1488): Thread RS:0;0c576948786c:37553 started persisting bucket cache to file 2024-11-12T21:52:36,398 DEBUG [RS:1;0c576948786c:33717 {}] bucket.BucketCache(1488): Thread RS:1;0c576948786c:33717 started persisting bucket cache to file 2024-11-12T21:52:36,398 DEBUG [RS:1;0c576948786c:33717 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T21:52:36,398 DEBUG [RS:0;0c576948786c:37553 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T21:52:36,398 DEBUG [RS:0;0c576948786c:37553 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T21:52:36,398 DEBUG [RS:1;0c576948786c:33717 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T21:52:36,401 DEBUG [RS:1;0c576948786c:33717 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T21:52:36,401 DEBUG [RS:0;0c576948786c:37553 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T21:52:36,401 DEBUG [RS:1;0c576948786c:33717 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731448356398 2024-11-12T21:52:36,401 DEBUG [RS:0;0c576948786c:37553 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731448356398 2024-11-12T21:52:36,401 DEBUG [RS:1;0c576948786c:33717 {}] bucket.BucketCache(1510): Thread RS:1;0c576948786c:33717 finished persisting bucket cache to file, renaming 2024-11-12T21:52:36,401 DEBUG [RS:0;0c576948786c:37553 {}] bucket.BucketCache(1510): Thread RS:0;0c576948786c:37553 finished persisting bucket cache to file, renaming 2024-11-12T21:52:36,401 WARN [RS:0;0c576948786c:37553 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-12T21:52:36,402 INFO [RS:0;0c576948786c:37553 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T21:52:36,402 INFO [RS:1;0c576948786c:33717 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T21:52:36,402 INFO [RS:0;0c576948786c:37553 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T21:52:36,402 INFO [RS:1;0c576948786c:33717 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T21:52:36,402 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T21:52:36,402 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T21:52:36,402 INFO [RS:1;0c576948786c:33717 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T21:52:36,402 INFO [RS:0;0c576948786c:37553 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T21:52:36,402 INFO [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(959): stopping server 0c576948786c,37553,1731448350870 2024-11-12T21:52:36,402 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(3091): Received CLOSE for a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,402 INFO [RS:0;0c576948786c:37553 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T21:52:36,402 INFO [RS:0;0c576948786c:37553 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0c576948786c:37553. 2024-11-12T21:52:36,402 DEBUG [RS:0;0c576948786c:37553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T21:52:36,402 DEBUG [RS:0;0c576948786c:37553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:36,403 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(959): stopping server 0c576948786c,33717,1731448351061 2024-11-12T21:52:36,403 INFO [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(976): stopping server 0c576948786c,37553,1731448350870; all regions closed. 2024-11-12T21:52:36,403 INFO [RS:1;0c576948786c:33717 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T21:52:36,403 INFO [RS:1;0c576948786c:33717 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0c576948786c:33717. 2024-11-12T21:52:36,403 DEBUG [RS:1;0c576948786c:33717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T21:52:36,403 DEBUG [RS:1;0c576948786c:33717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:36,403 INFO [RS:1;0c576948786c:33717 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T21:52:36,403 INFO [RS:1;0c576948786c:33717 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T21:52:36,403 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T21:52:36,403 INFO [RS:1;0c576948786c:33717 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T21:52:36,403 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T21:52:36,403 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a37659093bf81c8099642b42be820b91, disabling compactions & flushes 2024-11-12T21:52:36,404 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:36,404 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-12T21:52:36,404 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:36,404 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T21:52:36,404 DEBUG [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(1325): Online Regions={a37659093bf81c8099642b42be820b91=testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91., 1588230740=hbase:meta,,1.1588230740} 2024-11-12T21:52:36,404 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. after waiting 0 ms 2024-11-12T21:52:36,404 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T21:52:36,404 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:36,404 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T21:52:36,404 DEBUG [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a37659093bf81c8099642b42be820b91 2024-11-12T21:52:36,404 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T21:52:36,404 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T21:52:36,404 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T21:52:36,405 DEBUG [StoreFileCloser-a37659093bf81c8099642b42be820b91-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/family/1f5f4a985c3945308daf60635ab38b6a 2024-11-12T21:52:36,405 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-12T21:52:36,408 DEBUG [RS:0;0c576948786c:37553 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs 2024-11-12T21:52:36,408 INFO [RS:0;0c576948786c:37553 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0c576948786c%2C37553%2C1731448350870:(num 1731448353231) 2024-11-12T21:52:36,408 DEBUG [RS:0;0c576948786c:37553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:36,408 INFO [RS:0;0c576948786c:37553 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T21:52:36,408 INFO [RS:0;0c576948786c:37553 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T21:52:36,408 INFO [RS:0;0c576948786c:37553 {}] hbase.ChoreService(370): Chore service for: regionserver/0c576948786c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T21:52:36,409 INFO [RS:0;0c576948786c:37553 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T21:52:36,409 INFO [RS:0;0c576948786c:37553 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T21:52:36,409 INFO [regionserver/0c576948786c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T21:52:36,409 INFO [RS:0;0c576948786c:37553 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T21:52:36,409 INFO [RS:0;0c576948786c:37553 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T21:52:36,409 INFO [RS:0;0c576948786c:37553 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37553 2024-11-12T21:52:36,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c576948786c,37553,1731448350870 2024-11-12T21:52:36,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T21:52:36,412 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnRegionMove1731448353999/a37659093bf81c8099642b42be820b91/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-12T21:52:36,413 INFO [RS:0;0c576948786c:37553 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T21:52:36,413 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:36,413 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a37659093bf81c8099642b42be820b91: Waiting for close lock at 1731448356403Running coprocessor pre-close hooks at 1731448356403Disabling compacts and flushes for region at 1731448356403Disabling writes for close at 1731448356404 (+1 ms)Writing region close event to WAL at 1731448356405 (+1 ms)Running coprocessor post-close hooks at 1731448356413 (+8 ms)Closed at 1731448356413 2024-11-12T21:52:36,413 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91. 2024-11-12T21:52:36,414 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c576948786c,37553,1731448350870] 2024-11-12T21:52:36,415 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c576948786c,37553,1731448350870 already deleted, retry=false 2024-11-12T21:52:36,415 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c576948786c,37553,1731448350870 expired; onlineServers=1 2024-11-12T21:52:36,416 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3fd4d248d57140a796329afb425d42d0 2024-11-12T21:52:36,433 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/info/3fd4d248d57140a796329afb425d42d0 is 205, key is testBlockEvictionOnRegionMove1731448353999,,1731448354020.a37659093bf81c8099642b42be820b91./info:regioninfo/1731448356352/Put/seqid=0 2024-11-12T21:52:36,434 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/info/3fd4d248d57140a796329afb425d42d0 2024-11-12T21:52:36,443 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 515e680fd83045bdb8a26c6d462f87c1 2024-11-12T21:52:36,456 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/ns/515e680fd83045bdb8a26c6d462f87c1 is 43, key is default/ns:d/1731448353750/Put/seqid=0 2024-11-12T21:52:36,457 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/ns/515e680fd83045bdb8a26c6d462f87c1 2024-11-12T21:52:36,466 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e5e0d8190de947ed8727c82e79c8c0d4 2024-11-12T21:52:36,469 INFO [regionserver/0c576948786c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T21:52:36,472 INFO [regionserver/0c576948786c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T21:52:36,479 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/table/e5e0d8190de947ed8727c82e79c8c0d4 is 78, key is testBlockEvictionOnRegionMove1731448353999/table:state/1731448354432/Put/seqid=0 2024-11-12T21:52:36,480 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/table/e5e0d8190de947ed8727c82e79c8c0d4 2024-11-12T21:52:36,483 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/info/3fd4d248d57140a796329afb425d42d0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/3fd4d248d57140a796329afb425d42d0 2024-11-12T21:52:36,485 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/3fd4d248d57140a796329afb425d42d0, delay=991 ms 2024-11-12T21:52:36,487 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/3fd4d248d57140a796329afb425d42d0, entries=15, sequenceid=15, filesize=7.6 K 2024-11-12T21:52:36,487 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/ns/515e680fd83045bdb8a26c6d462f87c1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/515e680fd83045bdb8a26c6d462f87c1 2024-11-12T21:52:36,489 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/515e680fd83045bdb8a26c6d462f87c1, delay=948 ms 2024-11-12T21:52:36,491 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/515e680fd83045bdb8a26c6d462f87c1, entries=2, sequenceid=15, filesize=5.0 K 2024-11-12T21:52:36,492 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/table/e5e0d8190de947ed8727c82e79c8c0d4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/e5e0d8190de947ed8727c82e79c8c0d4 2024-11-12T21:52:36,494 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/e5e0d8190de947ed8727c82e79c8c0d4, delay=928 ms 2024-11-12T21:52:36,495 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/e5e0d8190de947ed8727c82e79c8c0d4, entries=2, sequenceid=15, filesize=5.3 K 2024-11-12T21:52:36,497 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false 2024-11-12T21:52:36,497 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T21:52:36,500 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/3fd4d248d57140a796329afb425d42d0 2024-11-12T21:52:36,501 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/515e680fd83045bdb8a26c6d462f87c1 2024-11-12T21:52:36,502 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/e5e0d8190de947ed8727c82e79c8c0d4 2024-11-12T21:52:36,507 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-12T21:52:36,507 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T21:52:36,508 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T21:52:36,508 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731448356404Running coprocessor pre-close hooks at 1731448356404Disabling compacts and flushes for region at 1731448356404Disabling writes for close at 1731448356404Obtaining lock to block concurrent updates at 1731448356405 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731448356405Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731448356405Flushing stores of hbase:meta,,1.1588230740 at 1731448356406 (+1 ms)Flushing 1588230740/info: creating writer at 1731448356406Flushing 1588230740/info: appending metadata at 1731448356432 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731448356432Flushing 1588230740/ns: creating writer at 1731448356436 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731448356455 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731448356455Flushing 1588230740/table: creating writer at 1731448356460 (+5 ms)Flushing 1588230740/table: appending metadata at 1731448356478 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731448356478Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d3efb9b: reopening flushed file at 1731448356483 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d1f598c: reopening flushed file at 1731448356487 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16f88c2c: reopening flushed file at 1731448356491 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false at 1731448356497 (+6 ms)Writing region close event to WAL at 1731448356502 (+5 ms)Running coprocessor post-close hooks at 1731448356507 (+5 ms)Closed at 1731448356508 (+1 ms) 2024-11-12T21:52:36,509 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T21:52:36,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:36,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x10035ec7eec0001, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:36,515 INFO [RS:0;0c576948786c:37553 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T21:52:36,516 INFO [RS:0;0c576948786c:37553 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c576948786c,37553,1731448350870; zookeeper connection closed. 2024-11-12T21:52:36,516 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7fa196bf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7fa196bf 2024-11-12T21:52:36,604 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(976): stopping server 0c576948786c,33717,1731448351061; all regions closed. 2024-11-12T21:52:36,607 DEBUG [RS:1;0c576948786c:33717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs 2024-11-12T21:52:36,607 INFO [RS:1;0c576948786c:33717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0c576948786c%2C33717%2C1731448351061.meta:.meta(num 1731448353605) 2024-11-12T21:52:36,609 DEBUG [RS:1;0c576948786c:33717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs 2024-11-12T21:52:36,609 INFO [RS:1;0c576948786c:33717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0c576948786c%2C33717%2C1731448351061:(num 1731448353230) 2024-11-12T21:52:36,609 DEBUG [RS:1;0c576948786c:33717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:36,609 INFO [RS:1;0c576948786c:33717 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T21:52:36,609 INFO [RS:1;0c576948786c:33717 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T21:52:36,609 INFO [RS:1;0c576948786c:33717 {}] hbase.ChoreService(370): Chore service for: regionserver/0c576948786c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T21:52:36,610 INFO [RS:1;0c576948786c:33717 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T21:52:36,610 INFO [regionserver/0c576948786c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T21:52:36,610 INFO [RS:1;0c576948786c:33717 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33717 2024-11-12T21:52:36,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c576948786c,33717,1731448351061 2024-11-12T21:52:36,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T21:52:36,612 INFO [RS:1;0c576948786c:33717 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T21:52:36,612 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c576948786c,33717,1731448351061] 2024-11-12T21:52:36,614 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c576948786c,33717,1731448351061 already deleted, retry=false 2024-11-12T21:52:36,614 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c576948786c,33717,1731448351061 expired; onlineServers=0 2024-11-12T21:52:36,614 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0c576948786c,45643,1731448349780' ***** 2024-11-12T21:52:36,614 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T21:52:36,614 INFO [M:0;0c576948786c:45643 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T21:52:36,614 INFO [M:0;0c576948786c:45643 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T21:52:36,614 DEBUG [M:0;0c576948786c:45643 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T21:52:36,615 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T21:52:36,615 DEBUG [M:0;0c576948786c:45643 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T21:52:36,615 DEBUG [master/0c576948786c:0:becomeActiveMaster-HFileCleaner.large.0-1731448352825 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c576948786c:0:becomeActiveMaster-HFileCleaner.large.0-1731448352825,5,FailOnTimeoutGroup] 2024-11-12T21:52:36,615 DEBUG [master/0c576948786c:0:becomeActiveMaster-HFileCleaner.small.0-1731448352829 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c576948786c:0:becomeActiveMaster-HFileCleaner.small.0-1731448352829,5,FailOnTimeoutGroup] 2024-11-12T21:52:36,615 INFO [M:0;0c576948786c:45643 {}] hbase.ChoreService(370): Chore service for: master/0c576948786c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T21:52:36,615 INFO [M:0;0c576948786c:45643 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T21:52:36,615 DEBUG [M:0;0c576948786c:45643 {}] master.HMaster(1795): Stopping service threads 2024-11-12T21:52:36,615 INFO [M:0;0c576948786c:45643 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T21:52:36,616 INFO [M:0;0c576948786c:45643 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T21:52:36,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T21:52:36,616 INFO [M:0;0c576948786c:45643 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T21:52:36,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:36,616 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T21:52:36,617 DEBUG [M:0;0c576948786c:45643 {}] zookeeper.ZKUtil(347): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T21:52:36,618 WARN [M:0;0c576948786c:45643 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T21:52:36,618 INFO [M:0;0c576948786c:45643 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/.lastflushedseqids 2024-11-12T21:52:36,632 INFO [M:0;0c576948786c:45643 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T21:52:36,632 INFO [M:0;0c576948786c:45643 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T21:52:36,632 DEBUG [M:0;0c576948786c:45643 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T21:52:36,632 INFO [M:0;0c576948786c:45643 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:36,632 DEBUG [M:0;0c576948786c:45643 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:36,632 DEBUG [M:0;0c576948786c:45643 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T21:52:36,632 DEBUG [M:0;0c576948786c:45643 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:36,633 INFO [M:0;0c576948786c:45643 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-12T21:52:36,643 DEBUG [M:0;0c576948786c:45643 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 46838e1f622c49269c8dc2662d02cfc5 2024-11-12T21:52:36,662 DEBUG [M:0;0c576948786c:45643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/46838e1f622c49269c8dc2662d02cfc5 is 82, key is hbase:meta,,1/info:regioninfo/1731448353676/Put/seqid=0 2024-11-12T21:52:36,663 INFO [M:0;0c576948786c:45643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/46838e1f622c49269c8dc2662d02cfc5 2024-11-12T21:52:36,672 DEBUG [M:0;0c576948786c:45643 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ff8a7c7c9f4d4dee85bf56880f0878bb 2024-11-12T21:52:36,687 DEBUG [M:0;0c576948786c:45643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ff8a7c7c9f4d4dee85bf56880f0878bb is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731448354437/Put/seqid=0 2024-11-12T21:52:36,688 INFO [M:0;0c576948786c:45643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ff8a7c7c9f4d4dee85bf56880f0878bb 2024-11-12T21:52:36,697 DEBUG [M:0;0c576948786c:45643 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2daec67251274f17932e3a31a51509cf 2024-11-12T21:52:36,710 DEBUG [M:0;0c576948786c:45643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2daec67251274f17932e3a31a51509cf is 69, key is 0c576948786c,33717,1731448351061/rs:state/1731448352955/Put/seqid=0 2024-11-12T21:52:36,711 INFO [M:0;0c576948786c:45643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2daec67251274f17932e3a31a51509cf 2024-11-12T21:52:36,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:36,713 INFO [RS:1;0c576948786c:33717 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T21:52:36,713 DEBUG [M:0;0c576948786c:45643 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/46838e1f622c49269c8dc2662d02cfc5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/46838e1f622c49269c8dc2662d02cfc5 2024-11-12T21:52:36,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33717-0x10035ec7eec0002, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:36,713 INFO [RS:1;0c576948786c:33717 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c576948786c,33717,1731448351061; zookeeper connection closed. 2024-11-12T21:52:36,714 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@69396918 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@69396918 2024-11-12T21:52:36,714 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-12T21:52:36,716 INFO [M:0;0c576948786c:45643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/46838e1f622c49269c8dc2662d02cfc5, entries=8, sequenceid=84, filesize=5.5 K 2024-11-12T21:52:36,717 DEBUG [M:0;0c576948786c:45643 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ff8a7c7c9f4d4dee85bf56880f0878bb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ff8a7c7c9f4d4dee85bf56880f0878bb 2024-11-12T21:52:36,719 INFO [M:0;0c576948786c:45643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ff8a7c7c9f4d4dee85bf56880f0878bb, entries=9, sequenceid=84, filesize=6.5 K 2024-11-12T21:52:36,720 DEBUG [M:0;0c576948786c:45643 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2daec67251274f17932e3a31a51509cf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2daec67251274f17932e3a31a51509cf 2024-11-12T21:52:36,723 INFO [M:0;0c576948786c:45643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2daec67251274f17932e3a31a51509cf, entries=2, sequenceid=84, filesize=5.1 K 2024-11-12T21:52:36,724 INFO [M:0;0c576948786c:45643 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false 2024-11-12T21:52:36,726 INFO [M:0;0c576948786c:45643 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:36,726 DEBUG [M:0;0c576948786c:45643 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731448356632Disabling compacts and flushes for region at 1731448356632Disabling writes for close at 1731448356632Obtaining lock to block concurrent updates at 1731448356633 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731448356633Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731448356634 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731448356634Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731448356634Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731448356661 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731448356661Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731448356666 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731448356686 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731448356686Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731448356691 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731448356709 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731448356709Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@698e6670: reopening flushed file at 1731448356713 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70933309: reopening flushed file at 1731448356716 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51199f84: reopening flushed file at 1731448356720 (+4 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false at 1731448356724 (+4 ms)Writing region close event to WAL at 1731448356726 (+2 ms)Closed at 1731448356726 2024-11-12T21:52:36,728 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T21:52:36,728 INFO [M:0;0c576948786c:45643 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T21:52:36,728 INFO [M:0;0c576948786c:45643 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45643 2024-11-12T21:52:36,728 INFO [M:0;0c576948786c:45643 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T21:52:36,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:36,830 INFO [M:0;0c576948786c:45643 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T21:52:36,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x10035ec7eec0000, quorum=127.0.0.1:54704, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:36,835 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T21:52:36,854 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T21:52:36,854 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:36,861 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731448356489 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731448354800 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/0c576948786c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0c576948786c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731448356485 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731448356338 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0c576948786c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/0c576948786c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=370 (was 324) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9448 (was 10191) 2024-11-12T21:52:36,867 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=370, ProcessCount=11, AvailableMemoryMB=9449 2024-11-12T21:52:36,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/hadoop-log-dir so I do NOT create it in target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990 2024-11-12T21:52:36,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/hadoop-tmp-dir so I do NOT create it in target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990 2024-11-12T21:52:36,867 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:36,870 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/cluster_fb4fa3f0-1213-02d6-28bd-b11fcb95c90f, deleteOnExit=true 2024-11-12T21:52:36,872 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/cluster_fb4fa3f0-1213-02d6-28bd-b11fcb95c90f/zookeeper_0, clientPort=54722, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/cluster_fb4fa3f0-1213-02d6-28bd-b11fcb95c90f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/cluster_fb4fa3f0-1213-02d6-28bd-b11fcb95c90f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T21:52:36,874 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54722 2024-11-12T21:52:36,886 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a with version=8 2024-11-12T21:52:36,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/hbase-staging 2024-11-12T21:52:36,888 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0c576948786c:0 server-side Connection retries=45 2024-11-12T21:52:36,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:36,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:36,889 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T21:52:36,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:36,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T21:52:36,889 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T21:52:36,890 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T21:52:36,891 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41893 2024-11-12T21:52:36,893 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41893 connecting to ZooKeeper ensemble=127.0.0.1:54722 2024-11-12T21:52:36,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:418930x0, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T21:52:36,897 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41893-0x10035ec9c760000 connected 2024-11-12T21:52:36,910 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:36,911 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:36,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:36,912 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a, hbase.cluster.distributed=false 2024-11-12T21:52:36,915 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T21:52:36,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41893 2024-11-12T21:52:36,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41893 2024-11-12T21:52:36,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41893 2024-11-12T21:52:36,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41893 2024-11-12T21:52:36,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41893 2024-11-12T21:52:36,936 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c576948786c:0 server-side Connection retries=45 2024-11-12T21:52:36,936 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:36,936 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:36,936 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T21:52:36,936 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:36,936 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T21:52:36,936 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T21:52:36,936 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T21:52:36,937 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33511 2024-11-12T21:52:36,938 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33511 connecting to ZooKeeper ensemble=127.0.0.1:54722 2024-11-12T21:52:36,938 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:36,939 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:36,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335110x0, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T21:52:36,942 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:36,942 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33511-0x10035ec9c760001 connected 2024-11-12T21:52:36,942 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T21:52:36,942 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.cache 2024-11-12T21:52:36,943 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T21:52:36,943 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T21:52:36,943 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T21:52:36,944 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T21:52:36,944 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T21:52:36,945 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T21:52:36,945 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T21:52:36,945 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.persistence is complete. 2024-11-12T21:52:36,945 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T21:52:36,946 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T21:52:36,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T21:52:36,949 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33511 2024-11-12T21:52:36,951 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33511 2024-11-12T21:52:36,953 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33511 2024-11-12T21:52:36,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33511 2024-11-12T21:52:36,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33511 2024-11-12T21:52:36,971 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c576948786c:0 server-side Connection retries=45 2024-11-12T21:52:36,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:36,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:36,971 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T21:52:36,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:36,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T21:52:36,971 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T21:52:36,972 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T21:52:36,972 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38285 2024-11-12T21:52:36,974 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38285 connecting to ZooKeeper ensemble=127.0.0.1:54722 2024-11-12T21:52:36,974 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:36,974 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:36,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382850x0, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T21:52:36,977 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:382850x0, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:36,977 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38285-0x10035ec9c760002 connected 2024-11-12T21:52:36,978 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T21:52:36,978 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.cache 2024-11-12T21:52:36,979 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T21:52:36,979 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T21:52:36,979 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T21:52:36,981 INFO [Thread-121 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T21:52:36,981 WARN [Thread-121 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T21:52:36,981 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T21:52:36,981 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T21:52:36,981 INFO [Thread-121 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.persistence is complete. 2024-11-12T21:52:36,981 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T21:52:36,982 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T21:52:36,983 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T21:52:36,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38285 2024-11-12T21:52:36,989 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38285 2024-11-12T21:52:36,989 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38285 2024-11-12T21:52:36,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38285 2024-11-12T21:52:36,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38285 2024-11-12T21:52:37,001 DEBUG [M:0;0c576948786c:41893 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0c576948786c:41893 2024-11-12T21:52:37,002 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0c576948786c,41893,1731448356888 2024-11-12T21:52:37,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:37,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:37,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:37,004 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0c576948786c,41893,1731448356888 2024-11-12T21:52:37,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T21:52:37,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T21:52:37,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,006 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T21:52:37,006 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0c576948786c,41893,1731448356888 from backup master directory 2024-11-12T21:52:37,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:37,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0c576948786c,41893,1731448356888 2024-11-12T21:52:37,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:37,014 WARN [master/0c576948786c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T21:52:37,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T21:52:37,014 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0c576948786c,41893,1731448356888 2024-11-12T21:52:37,015 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/hbase.id] with ID: 6c64d368-c945-496a-adf0-634c382b4f81 2024-11-12T21:52:37,016 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/.tmp/hbase.id 2024-11-12T21:52:37,021 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/hbase.id] 2024-11-12T21:52:37,054 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:37,054 INFO [master/0c576948786c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T21:52:37,056 INFO [master/0c576948786c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-12T21:52:37,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,063 INFO [master/0c576948786c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T21:52:37,064 INFO [master/0c576948786c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T21:52:37,065 INFO [master/0c576948786c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T21:52:37,083 INFO [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store 2024-11-12T21:52:37,088 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:37,088 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T21:52:37,088 INFO [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:37,089 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:37,089 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T21:52:37,089 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:37,089 INFO [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:37,089 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731448357088Disabling compacts and flushes for region at 1731448357088Disabling writes for close at 1731448357089 (+1 ms)Writing region close event to WAL at 1731448357089Closed at 1731448357089 2024-11-12T21:52:37,089 WARN [master/0c576948786c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/.initializing 2024-11-12T21:52:37,089 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/WALs/0c576948786c,41893,1731448356888 2024-11-12T21:52:37,095 INFO [master/0c576948786c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T21:52:37,098 INFO [master/0c576948786c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c576948786c%2C41893%2C1731448356888, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/WALs/0c576948786c,41893,1731448356888, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/oldWALs, maxLogs=10 2024-11-12T21:52:37,115 INFO [master/0c576948786c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/WALs/0c576948786c,41893,1731448356888/0c576948786c%2C41893%2C1731448356888.1731448357098 2024-11-12T21:52:37,116 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T21:52:37,116 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T21:52:37,116 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:37,116 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,116 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T21:52:37,123 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,123 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:37,123 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T21:52:37,127 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:37,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T21:52:37,131 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:37,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T21:52:37,134 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:37,135 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,135 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,135 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,135 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,135 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,136 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T21:52:37,136 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T21:52:37,141 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T21:52:37,141 INFO [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74175088, jitterRate=0.10529494285583496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T21:52:37,142 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731448357116Initializing all the Stores at 1731448357116Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448357117 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448357118 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448357118Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448357118Cleaning up temporary data from old regions at 1731448357135 (+17 ms)Region opened successfully at 1731448357142 (+7 ms) 2024-11-12T21:52:37,143 INFO [master/0c576948786c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T21:52:37,146 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16959735, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c576948786c/172.17.0.2:0 2024-11-12T21:52:37,148 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T21:52:37,148 INFO [master/0c576948786c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T21:52:37,148 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T21:52:37,148 INFO [master/0c576948786c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T21:52:37,148 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T21:52:37,149 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-12T21:52:37,149 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T21:52:37,151 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T21:52:37,152 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T21:52:37,153 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T21:52:37,153 INFO [master/0c576948786c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T21:52:37,154 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T21:52:37,155 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T21:52:37,155 INFO [master/0c576948786c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T21:52:37,156 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T21:52:37,157 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T21:52:37,158 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T21:52:37,159 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T21:52:37,161 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T21:52:37,162 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T21:52:37,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:37,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:37,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:37,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,164 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0c576948786c,41893,1731448356888, sessionid=0x10035ec9c760000, setting cluster-up flag (Was=false) 2024-11-12T21:52:37,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,171 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T21:52:37,172 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c576948786c,41893,1731448356888 2024-11-12T21:52:37,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,178 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T21:52:37,180 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c576948786c,41893,1731448356888 2024-11-12T21:52:37,181 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T21:52:37,182 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T21:52:37,182 INFO [master/0c576948786c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T21:52:37,182 INFO [master/0c576948786c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T21:52:37,183 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0c576948786c,41893,1731448356888 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T21:52:37,185 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0c576948786c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T21:52:37,185 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0c576948786c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T21:52:37,185 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0c576948786c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T21:52:37,185 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0c576948786c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T21:52:37,185 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0c576948786c:0, corePoolSize=10, maxPoolSize=10 2024-11-12T21:52:37,185 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,185 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0c576948786c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T21:52:37,186 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,187 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T21:52:37,187 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T21:52:37,188 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,188 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T21:52:37,189 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731448387189 2024-11-12T21:52:37,189 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T21:52:37,189 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T21:52:37,189 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T21:52:37,189 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T21:52:37,189 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T21:52:37,190 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T21:52:37,190 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,192 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T21:52:37,192 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T21:52:37,192 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T21:52:37,193 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T21:52:37,193 INFO [master/0c576948786c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T21:52:37,193 INFO [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(746): ClusterId : 6c64d368-c945-496a-adf0-634c382b4f81 2024-11-12T21:52:37,193 DEBUG [RS:0;0c576948786c:33511 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T21:52:37,194 INFO [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(746): ClusterId : 6c64d368-c945-496a-adf0-634c382b4f81 2024-11-12T21:52:37,194 DEBUG [RS:1;0c576948786c:38285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T21:52:37,195 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0c576948786c:0:becomeActiveMaster-HFileCleaner.large.0-1731448357193,5,FailOnTimeoutGroup] 2024-11-12T21:52:37,195 DEBUG [RS:0;0c576948786c:33511 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T21:52:37,195 DEBUG [RS:0;0c576948786c:33511 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T21:52:37,196 DEBUG [RS:1;0c576948786c:38285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T21:52:37,196 DEBUG [RS:1;0c576948786c:38285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T21:52:37,197 DEBUG [RS:0;0c576948786c:33511 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T21:52:37,198 DEBUG [RS:1;0c576948786c:38285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T21:52:37,198 DEBUG [RS:1;0c576948786c:38285 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe44d69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c576948786c/172.17.0.2:0 2024-11-12T21:52:37,198 DEBUG [RS:0;0c576948786c:33511 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ad3ae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c576948786c/172.17.0.2:0 2024-11-12T21:52:37,199 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0c576948786c:0:becomeActiveMaster-HFileCleaner.small.0-1731448357195,5,FailOnTimeoutGroup] 2024-11-12T21:52:37,199 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,199 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T21:52:37,199 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,200 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,212 DEBUG [RS:0;0c576948786c:33511 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0c576948786c:33511 2024-11-12T21:52:37,212 INFO [RS:0;0c576948786c:33511 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T21:52:37,212 INFO [RS:0;0c576948786c:33511 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T21:52:37,212 DEBUG [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T21:52:37,213 INFO [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c576948786c,41893,1731448356888 with port=33511, startcode=1731448356935 2024-11-12T21:52:37,213 DEBUG [RS:0;0c576948786c:33511 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T21:52:37,216 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60871, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T21:52:37,216 DEBUG [RS:1;0c576948786c:38285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0c576948786c:38285 2024-11-12T21:52:37,216 INFO [RS:1;0c576948786c:38285 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T21:52:37,216 INFO [RS:1;0c576948786c:38285 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T21:52:37,216 DEBUG [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T21:52:37,216 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41893 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c576948786c,33511,1731448356935 2024-11-12T21:52:37,216 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41893 {}] master.ServerManager(517): Registering regionserver=0c576948786c,33511,1731448356935 2024-11-12T21:52:37,218 INFO [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c576948786c,41893,1731448356888 with port=38285, startcode=1731448356970 2024-11-12T21:52:37,218 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T21:52:37,218 DEBUG [RS:1;0c576948786c:38285 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T21:52:37,218 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a 2024-11-12T21:52:37,218 DEBUG [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a 2024-11-12T21:52:37,218 DEBUG [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T21:52:37,218 DEBUG [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T21:52:37,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T21:52:37,220 DEBUG [RS:0;0c576948786c:33511 {}] zookeeper.ZKUtil(111): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c576948786c,33511,1731448356935 2024-11-12T21:52:37,220 WARN [RS:0;0c576948786c:33511 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T21:52:37,220 INFO [RS:0;0c576948786c:33511 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T21:52:37,220 DEBUG [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33511,1731448356935 2024-11-12T21:52:37,222 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c576948786c,33511,1731448356935] 2024-11-12T21:52:37,226 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44537, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T21:52:37,226 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41893 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c576948786c,38285,1731448356970 2024-11-12T21:52:37,226 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41893 {}] master.ServerManager(517): Registering regionserver=0c576948786c,38285,1731448356970 2024-11-12T21:52:37,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:37,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T21:52:37,229 INFO [RS:0;0c576948786c:33511 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T21:52:37,231 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T21:52:37,232 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:37,232 INFO [RS:0;0c576948786c:33511 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T21:52:37,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T21:52:37,232 INFO [RS:0;0c576948786c:33511 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T21:52:37,232 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,233 INFO [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T21:52:37,233 DEBUG [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a 2024-11-12T21:52:37,233 DEBUG [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T21:52:37,233 DEBUG [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T21:52:37,233 INFO [RS:0;0c576948786c:33511 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T21:52:37,233 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c576948786c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,234 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,235 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c576948786c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T21:52:37,235 DEBUG [RS:0;0c576948786c:33511 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c576948786c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T21:52:37,235 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,235 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,235 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,235 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,235 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,235 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,33511,1731448356935-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T21:52:37,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T21:52:37,236 DEBUG [RS:1;0c576948786c:38285 {}] zookeeper.ZKUtil(111): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c576948786c,38285,1731448356970 2024-11-12T21:52:37,237 WARN [RS:1;0c576948786c:38285 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T21:52:37,237 INFO [RS:1;0c576948786c:38285 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T21:52:37,237 DEBUG [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,38285,1731448356970 2024-11-12T21:52:37,237 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c576948786c,38285,1731448356970] 2024-11-12T21:52:37,242 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T21:52:37,242 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:37,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T21:52:37,245 INFO [RS:1;0c576948786c:38285 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T21:52:37,247 INFO [RS:1;0c576948786c:38285 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T21:52:37,249 INFO [RS:1;0c576948786c:38285 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T21:52:37,249 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,252 INFO [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T21:52:37,252 INFO [RS:1;0c576948786c:38285 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T21:52:37,252 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,252 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T21:52:37,253 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,253 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,253 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,253 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:37,253 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,253 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T21:52:37,253 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c576948786c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T21:52:37,253 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,253 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,253 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,254 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,254 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,254 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:37,254 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c576948786c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T21:52:37,254 DEBUG [RS:1;0c576948786c:38285 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c576948786c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T21:52:37,255 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,255 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,255 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,255 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,255 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,255 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,38285,1731448356970-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T21:52:37,258 INFO [RS:0;0c576948786c:33511 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T21:52:37,258 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,33511,1731448356935-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,258 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,258 INFO [RS:0;0c576948786c:33511 {}] regionserver.Replication(171): 0c576948786c,33511,1731448356935 started 2024-11-12T21:52:37,258 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T21:52:37,259 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:37,259 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T21:52:37,259 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740 2024-11-12T21:52:37,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740 2024-11-12T21:52:37,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T21:52:37,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T21:52:37,260 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T21:52:37,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T21:52:37,267 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T21:52:37,267 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67083716, jitterRate=-3.7473440170288086E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T21:52:37,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731448357227Initializing all the Stores at 1731448357227Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448357227Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448357228 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448357228Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448357228Cleaning up temporary data from old regions at 1731448357260 (+32 ms)Region opened successfully at 1731448357268 (+8 ms) 2024-11-12T21:52:37,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T21:52:37,268 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T21:52:37,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T21:52:37,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T21:52:37,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T21:52:37,269 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T21:52:37,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731448357268Disabling compacts and flushes for region at 1731448357268Disabling writes for close at 1731448357268Writing region close event to WAL at 1731448357269 (+1 ms)Closed at 1731448357269 2024-11-12T21:52:37,270 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T21:52:37,270 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T21:52:37,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T21:52:37,271 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T21:52:37,272 INFO [RS:1;0c576948786c:38285 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T21:52:37,272 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T21:52:37,272 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,38285,1731448356970-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,272 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,273 INFO [RS:1;0c576948786c:38285 {}] regionserver.Replication(171): 0c576948786c,38285,1731448356970 started 2024-11-12T21:52:37,275 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,275 INFO [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(1482): Serving as 0c576948786c,33511,1731448356935, RpcServer on 0c576948786c/172.17.0.2:33511, sessionid=0x10035ec9c760001 2024-11-12T21:52:37,275 DEBUG [RS:0;0c576948786c:33511 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T21:52:37,275 DEBUG [RS:0;0c576948786c:33511 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c576948786c,33511,1731448356935 2024-11-12T21:52:37,275 DEBUG [RS:0;0c576948786c:33511 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c576948786c,33511,1731448356935' 2024-11-12T21:52:37,275 DEBUG [RS:0;0c576948786c:33511 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T21:52:37,276 DEBUG [RS:0;0c576948786c:33511 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T21:52:37,276 DEBUG [RS:0;0c576948786c:33511 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T21:52:37,276 DEBUG [RS:0;0c576948786c:33511 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T21:52:37,276 DEBUG [RS:0;0c576948786c:33511 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c576948786c,33511,1731448356935 2024-11-12T21:52:37,276 DEBUG [RS:0;0c576948786c:33511 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c576948786c,33511,1731448356935' 2024-11-12T21:52:37,276 DEBUG [RS:0;0c576948786c:33511 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T21:52:37,277 DEBUG [RS:0;0c576948786c:33511 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T21:52:37,277 DEBUG [RS:0;0c576948786c:33511 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T21:52:37,277 INFO [RS:0;0c576948786c:33511 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T21:52:37,278 INFO [RS:0;0c576948786c:33511 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T21:52:37,287 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,287 INFO [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(1482): Serving as 0c576948786c,38285,1731448356970, RpcServer on 0c576948786c/172.17.0.2:38285, sessionid=0x10035ec9c760002 2024-11-12T21:52:37,287 DEBUG [RS:1;0c576948786c:38285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T21:52:37,287 DEBUG [RS:1;0c576948786c:38285 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c576948786c,38285,1731448356970 2024-11-12T21:52:37,287 DEBUG [RS:1;0c576948786c:38285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c576948786c,38285,1731448356970' 2024-11-12T21:52:37,287 DEBUG [RS:1;0c576948786c:38285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T21:52:37,288 DEBUG [RS:1;0c576948786c:38285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T21:52:37,288 DEBUG [RS:1;0c576948786c:38285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T21:52:37,288 DEBUG [RS:1;0c576948786c:38285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T21:52:37,288 DEBUG [RS:1;0c576948786c:38285 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c576948786c,38285,1731448356970 2024-11-12T21:52:37,288 DEBUG [RS:1;0c576948786c:38285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c576948786c,38285,1731448356970' 2024-11-12T21:52:37,288 DEBUG [RS:1;0c576948786c:38285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T21:52:37,289 DEBUG [RS:1;0c576948786c:38285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T21:52:37,289 DEBUG [RS:1;0c576948786c:38285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T21:52:37,290 INFO [RS:1;0c576948786c:38285 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T21:52:37,290 INFO [RS:1;0c576948786c:38285 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T21:52:37,379 INFO [RS:0;0c576948786c:33511 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T21:52:37,381 INFO [RS:0;0c576948786c:33511 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0c576948786c%2C33511%2C1731448356935, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33511,1731448356935, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs, maxLogs=55 2024-11-12T21:52:37,391 INFO [RS:1;0c576948786c:38285 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T21:52:37,392 INFO [RS:1;0c576948786c:38285 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0c576948786c%2C38285%2C1731448356970, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,38285,1731448356970, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs, maxLogs=55 2024-11-12T21:52:37,408 INFO [RS:1;0c576948786c:38285 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,38285,1731448356970/0c576948786c%2C38285%2C1731448356970.1731448357393 2024-11-12T21:52:37,409 DEBUG [RS:1;0c576948786c:38285 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T21:52:37,411 INFO [RS:0;0c576948786c:33511 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33511,1731448356935/0c576948786c%2C33511%2C1731448356935.1731448357386 2024-11-12T21:52:37,412 DEBUG [RS:0;0c576948786c:33511 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T21:52:37,422 DEBUG [0c576948786c:41893 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-12T21:52:37,423 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(204): Hosts are {0c576948786c=0} racks are {/default-rack=0} 2024-11-12T21:52:37,423 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T21:52:37,423 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T21:52:37,423 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T21:52:37,424 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T21:52:37,424 INFO [0c576948786c:41893 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T21:52:37,424 INFO [0c576948786c:41893 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T21:52:37,424 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T21:52:37,424 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0c576948786c,33511,1731448356935 2024-11-12T21:52:37,425 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c576948786c,33511,1731448356935, state=OPENING 2024-11-12T21:52:37,426 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T21:52:37,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:37,428 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T21:52:37,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0c576948786c,33511,1731448356935}] 2024-11-12T21:52:37,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:37,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:37,429 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:37,582 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T21:52:37,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43625, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T21:52:37,589 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T21:52:37,590 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T21:52:37,590 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T21:52:37,592 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0c576948786c%2C33511%2C1731448356935.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33511,1731448356935, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs, maxLogs=55 2024-11-12T21:52:37,609 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33511,1731448356935/0c576948786c%2C33511%2C1731448356935.meta.1731448357593.meta 2024-11-12T21:52:37,609 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T21:52:37,610 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T21:52:37,610 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T21:52:37,610 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T21:52:37,610 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T21:52:37,610 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T21:52:37,610 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:37,610 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T21:52:37,611 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T21:52:37,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T21:52:37,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T21:52:37,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:37,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T21:52:37,613 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T21:52:37,613 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:37,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T21:52:37,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T21:52:37,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:37,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T21:52:37,615 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T21:52:37,615 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:37,615 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T21:52:37,616 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740 2024-11-12T21:52:37,616 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740 2024-11-12T21:52:37,616 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T21:52:37,616 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T21:52:37,616 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T21:52:37,617 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T21:52:37,617 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69979644, jitterRate=0.042777955532073975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T21:52:37,617 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T21:52:37,618 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731448357611Writing region info on filesystem at 1731448357611Initializing all the Stores at 1731448357611Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448357611Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448357611Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448357611Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448357611Cleaning up temporary data from old regions at 1731448357616 (+5 ms)Running coprocessor post-open hooks at 1731448357617 (+1 ms)Region opened successfully at 1731448357618 (+1 ms) 2024-11-12T21:52:37,619 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731448357581 2024-11-12T21:52:37,621 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T21:52:37,621 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T21:52:37,622 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0c576948786c,33511,1731448356935 2024-11-12T21:52:37,622 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c576948786c,33511,1731448356935, state=OPEN 2024-11-12T21:52:37,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:37,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:37,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:37,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:37,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:37,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:37,624 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0c576948786c,33511,1731448356935 2024-11-12T21:52:37,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T21:52:37,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0c576948786c,33511,1731448356935 in 196 msec 2024-11-12T21:52:37,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T21:52:37,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 357 msec 2024-11-12T21:52:37,630 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T21:52:37,630 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T21:52:37,633 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T21:52:37,633 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c576948786c,33511,1731448356935, seqNum=-1] 2024-11-12T21:52:37,634 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T21:52:37,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59107, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T21:52:37,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 459 msec 2024-11-12T21:52:37,642 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731448357642, completionTime=-1 2024-11-12T21:52:37,643 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-12T21:52:37,643 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T21:52:37,645 INFO [master/0c576948786c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T21:52:37,646 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731448417646 2024-11-12T21:52:37,646 INFO [master/0c576948786c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731448477646 2024-11-12T21:52:37,646 INFO [master/0c576948786c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-12T21:52:37,646 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-12T21:52:37,647 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,41893,1731448356888-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,647 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,41893,1731448356888-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,647 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,41893,1731448356888-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,647 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0c576948786c:41893, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,647 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,648 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,650 DEBUG [master/0c576948786c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T21:52:37,653 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.639sec 2024-11-12T21:52:37,654 INFO [master/0c576948786c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T21:52:37,654 INFO [master/0c576948786c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T21:52:37,654 INFO [master/0c576948786c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T21:52:37,654 INFO [master/0c576948786c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T21:52:37,654 INFO [master/0c576948786c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T21:52:37,654 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,41893,1731448356888-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T21:52:37,654 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,41893,1731448356888-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T21:52:37,655 DEBUG [master/0c576948786c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T21:52:37,655 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T21:52:37,655 INFO [master/0c576948786c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,41893,1731448356888-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:37,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c5fbb56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T21:52:37,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0c576948786c,41893,-1 for getting cluster id 2024-11-12T21:52:37,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T21:52:37,696 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6c64d368-c945-496a-adf0-634c382b4f81' 2024-11-12T21:52:37,697 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T21:52:37,697 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6c64d368-c945-496a-adf0-634c382b4f81" 2024-11-12T21:52:37,697 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1ed429, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T21:52:37,697 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0c576948786c,41893,-1] 2024-11-12T21:52:37,698 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T21:52:37,698 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:37,700 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T21:52:37,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b86ba92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T21:52:37,702 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T21:52:37,703 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c576948786c,33511,1731448356935, seqNum=-1] 2024-11-12T21:52:37,704 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T21:52:37,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T21:52:37,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0c576948786c,41893,1731448356888 2024-11-12T21:52:37,709 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T21:52:37,710 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 0c576948786c,41893,1731448356888 2024-11-12T21:52:37,710 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61194eaa 2024-11-12T21:52:37,711 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T21:52:37,713 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T21:52:37,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41893 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731448357708', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T21:52:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41893 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731448357708 2024-11-12T21:52:37,716 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731448357708 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T21:52:37,716 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:37,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41893 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731448357708" procId is: 4 2024-11-12T21:52:37,717 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731448357708 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T21:52:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41893 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T21:52:37,726 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731448357708-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 085504eca23ebcdf3979bbbfc20b190c, NAME => 'testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731448357708', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a 2024-11-12T21:52:37,733 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731448357708-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:37,734 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731448357708-pool-0 {}] regionserver.HRegion(1722): Closing 085504eca23ebcdf3979bbbfc20b190c, disabling compactions & flushes 2024-11-12T21:52:37,734 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731448357708-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:37,734 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731448357708-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:37,734 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731448357708-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. after waiting 0 ms 2024-11-12T21:52:37,734 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731448357708-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:37,734 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731448357708-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:37,734 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731448357708-pool-0 {}] regionserver.HRegion(1676): Region close journal for 085504eca23ebcdf3979bbbfc20b190c: Waiting for close lock at 1731448357734Disabling compacts and flushes for region at 1731448357734Disabling writes for close at 1731448357734Writing region close event to WAL at 1731448357734Closed at 1731448357734 2024-11-12T21:52:37,735 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731448357708 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T21:52:37,736 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731448357735"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731448357735"}]},"ts":"1731448357735"} 2024-11-12T21:52:37,738 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T21:52:37,739 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731448357708 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T21:52:37,739 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731448357708","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731448357739"}]},"ts":"1731448357739"} 2024-11-12T21:52:37,742 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731448357708, state=ENABLING in hbase:meta 2024-11-12T21:52:37,742 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0c576948786c=0} racks are {/default-rack=0} 2024-11-12T21:52:37,742 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T21:52:37,742 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T21:52:37,742 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T21:52:37,742 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T21:52:37,742 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T21:52:37,742 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T21:52:37,743 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T21:52:37,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731448357708, region=085504eca23ebcdf3979bbbfc20b190c, ASSIGN}] 2024-11-12T21:52:37,744 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731448357708, region=085504eca23ebcdf3979bbbfc20b190c, ASSIGN 2024-11-12T21:52:37,745 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731448357708, region=085504eca23ebcdf3979bbbfc20b190c, ASSIGN; state=OFFLINE, location=0c576948786c,33511,1731448356935; forceNewPlan=false, retain=false 2024-11-12T21:52:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41893 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T21:52:37,896 INFO [0c576948786c:41893 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T21:52:37,896 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=085504eca23ebcdf3979bbbfc20b190c, regionState=OPENING, regionLocation=0c576948786c,33511,1731448356935 2024-11-12T21:52:37,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731448357708, region=085504eca23ebcdf3979bbbfc20b190c, ASSIGN because future has completed 2024-11-12T21:52:37,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 085504eca23ebcdf3979bbbfc20b190c, server=0c576948786c,33511,1731448356935}] 2024-11-12T21:52:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41893 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T21:52:38,056 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:38,057 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 085504eca23ebcdf3979bbbfc20b190c, NAME => 'testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c.', STARTKEY => '', ENDKEY => ''} 2024-11-12T21:52:38,057 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731448357708 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,057 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:38,057 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,057 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,058 INFO [StoreOpener-085504eca23ebcdf3979bbbfc20b190c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,062 INFO [StoreOpener-085504eca23ebcdf3979bbbfc20b190c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 085504eca23ebcdf3979bbbfc20b190c columnFamilyName family 2024-11-12T21:52:38,062 DEBUG [StoreOpener-085504eca23ebcdf3979bbbfc20b190c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:38,062 INFO [StoreOpener-085504eca23ebcdf3979bbbfc20b190c-1 {}] regionserver.HStore(327): Store=085504eca23ebcdf3979bbbfc20b190c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:38,062 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,063 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,063 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,063 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,063 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,064 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,069 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T21:52:38,069 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 085504eca23ebcdf3979bbbfc20b190c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71764185, jitterRate=0.06936968863010406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T21:52:38,070 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:38,070 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 085504eca23ebcdf3979bbbfc20b190c: Running coprocessor pre-open hook at 1731448358058Writing region info on filesystem at 1731448358058Initializing all the Stores at 1731448358058Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448358058Cleaning up temporary data from old regions at 1731448358063 (+5 ms)Running coprocessor post-open hooks at 1731448358070 (+7 ms)Region opened successfully at 1731448358070 2024-11-12T21:52:38,071 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c., pid=6, masterSystemTime=1731448358054 2024-11-12T21:52:38,074 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:38,074 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:38,074 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=085504eca23ebcdf3979bbbfc20b190c, regionState=OPEN, openSeqNum=2, regionLocation=0c576948786c,33511,1731448356935 2024-11-12T21:52:38,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 085504eca23ebcdf3979bbbfc20b190c, server=0c576948786c,33511,1731448356935 because future has completed 2024-11-12T21:52:38,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T21:52:38,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 085504eca23ebcdf3979bbbfc20b190c, server=0c576948786c,33511,1731448356935 in 177 msec 2024-11-12T21:52:38,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T21:52:38,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731448357708, region=085504eca23ebcdf3979bbbfc20b190c, ASSIGN in 338 msec 2024-11-12T21:52:38,084 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731448357708 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T21:52:38,084 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731448357708","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731448358084"}]},"ts":"1731448358084"} 2024-11-12T21:52:38,086 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731448357708, state=ENABLED in hbase:meta 2024-11-12T21:52:38,087 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731448357708 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T21:52:38,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731448357708 in 373 msec 2024-11-12T21:52:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41893 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T21:52:38,345 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731448357708 completed 2024-11-12T21:52:38,345 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731448357708 get assigned. Timeout = 60000ms 2024-11-12T21:52:38,346 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T21:52:38,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731448357708 assigned to meta. Checking AM states. 2024-11-12T21:52:38,351 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T21:52:38,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731448357708 assigned. 2024-11-12T21:52:38,355 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731448357708', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c., hostname=0c576948786c,33511,1731448356935, seqNum=2] 2024-11-12T21:52:38,359 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 085504eca23ebcdf3979bbbfc20b190c 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-12T21:52:38,367 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on de815d09118f4740ac31ed2750c5e564 2024-11-12T21:52:38,380 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/.tmp/family/de815d09118f4740ac31ed2750c5e564 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-12T21:52:38,380 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/.tmp/family/de815d09118f4740ac31ed2750c5e564 2024-11-12T21:52:38,384 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/.tmp/family/de815d09118f4740ac31ed2750c5e564 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564 2024-11-12T21:52:38,385 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564, delay=953 ms 2024-11-12T21:52:38,387 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564, entries=2, sequenceid=6, filesize=4.9 K 2024-11-12T21:52:38,387 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 085504eca23ebcdf3979bbbfc20b190c in 28ms, sequenceid=6, compaction requested=false 2024-11-12T21:52:38,388 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731448357708' 2024-11-12T21:52:38,388 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 085504eca23ebcdf3979bbbfc20b190c: 2024-11-12T21:52:39,145 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T21:52:39,339 DEBUG [hfile-prefetch-1731448356338 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key de815d09118f4740ac31ed2750c5e564_0. 2024-11-12T21:52:39,340 DEBUG [hfile-prefetch-1731448356338 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key de815d09118f4740ac31ed2750c5e564_117. 2024-11-12T21:52:39,340 DEBUG [hfile-prefetch-1731448356338 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564, with total blocks 2, and data blocks 1 2024-11-12T21:52:39,341 DEBUG [hfile-prefetch-1731448356338 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-12T21:52:39,341 DEBUG [hfile-prefetch-1731448356338 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-12T21:52:39,341 DEBUG [hfile-prefetch-1731448356338 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564. Will sleep 100ms and try the verification again. 2024-11-12T21:52:39,346 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T21:52:39,346 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T21:52:39,346 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T21:52:39,347 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T21:52:39,351 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-12T21:52:39,351 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731448359346 2024-11-12T21:52:39,352 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T21:52:39,389 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;0c576948786c:33511,5,FailOnTimeoutGroup] 2024-11-12T21:52:39,390 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c576948786c,33511,1731448356935' ***** 2024-11-12T21:52:39,390 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-12T21:52:39,394 INFO [RS:0;0c576948786c:33511 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T21:52:39,395 INFO [RS:0;0c576948786c:33511 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.persistence 2024-11-12T21:52:39,395 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T21:52:39,395 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T21:52:39,395 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T21:52:39,395 INFO [RS:0;0c576948786c:33511 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T21:52:39,396 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T21:52:39,406 DEBUG [RS:0;0c576948786c:33511 {}] bucket.BucketCache(1488): Thread RS:0;0c576948786c:33511 started persisting bucket cache to file 2024-11-12T21:52:39,406 DEBUG [RS:0;0c576948786c:33511 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T21:52:39,406 DEBUG [RS:0;0c576948786c:33511 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T21:52:39,409 DEBUG [RS:0;0c576948786c:33511 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-12T21:52:39,409 DEBUG [RS:0;0c576948786c:33511 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731448359406 2024-11-12T21:52:39,409 DEBUG [RS:0;0c576948786c:33511 {}] bucket.BucketCache(1510): Thread RS:0;0c576948786c:33511 finished persisting bucket cache to file, renaming 2024-11-12T21:52:39,410 INFO [RS:0;0c576948786c:33511 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T21:52:39,410 INFO [RS:0;0c576948786c:33511 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T21:52:39,410 INFO [RS:0;0c576948786c:33511 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T21:52:39,410 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T21:52:39,410 INFO [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(3091): Received CLOSE for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:39,411 INFO [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(959): stopping server 0c576948786c,33511,1731448356935 2024-11-12T21:52:39,411 INFO [RS:0;0c576948786c:33511 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T21:52:39,411 INFO [RS:0;0c576948786c:33511 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0c576948786c:33511. 2024-11-12T21:52:39,411 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T21:52:39,411 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 085504eca23ebcdf3979bbbfc20b190c, disabling compactions & flushes 2024-11-12T21:52:39,411 DEBUG [RS:0;0c576948786c:33511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T21:52:39,411 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:39,411 DEBUG [RS:0;0c576948786c:33511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:39,411 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:39,411 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. after waiting 0 ms 2024-11-12T21:52:39,411 INFO [RS:0;0c576948786c:33511 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T21:52:39,411 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:39,412 INFO [RS:0;0c576948786c:33511 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T21:52:39,412 INFO [RS:0;0c576948786c:33511 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T21:52:39,412 INFO [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T21:52:39,412 INFO [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-12T21:52:39,412 DEBUG [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(1325): Online Regions={085504eca23ebcdf3979bbbfc20b190c=testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c., 1588230740=hbase:meta,,1.1588230740} 2024-11-12T21:52:39,412 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T21:52:39,413 DEBUG [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(1351): Waiting on 085504eca23ebcdf3979bbbfc20b190c, 1588230740 2024-11-12T21:52:39,413 DEBUG [StoreFileCloser-085504eca23ebcdf3979bbbfc20b190c-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564 2024-11-12T21:52:39,413 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T21:52:39,413 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T21:52:39,413 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T21:52:39,413 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T21:52:39,413 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T21:52:39,413 WARN [hfile-prefetch-1731448356338 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-12T21:52:39,414 DEBUG [hfile-prefetch-1731448356338 {}] hfile.PrefetchExecutor(121): Prefetch completed for de815d09118f4740ac31ed2750c5e564 2024-11-12T21:52:39,414 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-12T21:52:39,418 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-12T21:52:39,419 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:39,419 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 085504eca23ebcdf3979bbbfc20b190c: Waiting for close lock at 1731448359411Running coprocessor pre-close hooks at 1731448359411Disabling compacts and flushes for region at 1731448359411Disabling writes for close at 1731448359411Writing region close event to WAL at 1731448359413 (+2 ms)Running coprocessor post-close hooks at 1731448359419 (+6 ms)Closed at 1731448359419 2024-11-12T21:52:39,419 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:39,423 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b21dccd548d947c79707d991ac191030 2024-11-12T21:52:39,436 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/info/b21dccd548d947c79707d991ac191030 is 209, key is testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c./info:regioninfo/1731448358074/Put/seqid=0 2024-11-12T21:52:39,436 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/info/b21dccd548d947c79707d991ac191030 2024-11-12T21:52:39,443 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2e69984983754390aab7355bd2268ada 2024-11-12T21:52:39,443 INFO [regionserver/0c576948786c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T21:52:39,456 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/ns/2e69984983754390aab7355bd2268ada is 43, key is default/ns:d/1731448357636/Put/seqid=0 2024-11-12T21:52:39,457 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/ns/2e69984983754390aab7355bd2268ada 2024-11-12T21:52:39,463 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 64293acb0f7e48138850a2afe9e4111c 2024-11-12T21:52:39,476 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/table/64293acb0f7e48138850a2afe9e4111c is 80, key is testBlockEvictionOnGracefulStop1731448357708/table:state/1731448358084/Put/seqid=0 2024-11-12T21:52:39,477 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/table/64293acb0f7e48138850a2afe9e4111c 2024-11-12T21:52:39,479 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/info/b21dccd548d947c79707d991ac191030 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/b21dccd548d947c79707d991ac191030 2024-11-12T21:52:39,480 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/b21dccd548d947c79707d991ac191030, delay=905 ms 2024-11-12T21:52:39,481 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/b21dccd548d947c79707d991ac191030, entries=10, sequenceid=11, filesize=7.0 K 2024-11-12T21:52:39,482 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/ns/2e69984983754390aab7355bd2268ada as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/2e69984983754390aab7355bd2268ada 2024-11-12T21:52:39,483 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/2e69984983754390aab7355bd2268ada, delay=905 ms 2024-11-12T21:52:39,484 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/2e69984983754390aab7355bd2268ada, entries=2, sequenceid=11, filesize=5.0 K 2024-11-12T21:52:39,484 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/table/64293acb0f7e48138850a2afe9e4111c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/64293acb0f7e48138850a2afe9e4111c 2024-11-12T21:52:39,485 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/64293acb0f7e48138850a2afe9e4111c, delay=923 ms 2024-11-12T21:52:39,486 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/64293acb0f7e48138850a2afe9e4111c, entries=2, sequenceid=11, filesize=5.3 K 2024-11-12T21:52:39,487 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=11, compaction requested=false 2024-11-12T21:52:39,487 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/b21dccd548d947c79707d991ac191030 2024-11-12T21:52:39,488 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/2e69984983754390aab7355bd2268ada 2024-11-12T21:52:39,488 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/64293acb0f7e48138850a2afe9e4111c 2024-11-12T21:52:39,492 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-12T21:52:39,492 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T21:52:39,493 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T21:52:39,493 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731448359412Running coprocessor pre-close hooks at 1731448359413 (+1 ms)Disabling compacts and flushes for region at 1731448359413Disabling writes for close at 1731448359413Obtaining lock to block concurrent updates at 1731448359414 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731448359414Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731448359414Flushing stores of hbase:meta,,1.1588230740 at 1731448359415 (+1 ms)Flushing 1588230740/info: creating writer at 1731448359415Flushing 1588230740/info: appending metadata at 1731448359435 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731448359435Flushing 1588230740/ns: creating writer at 1731448359438 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731448359455 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731448359455Flushing 1588230740/table: creating writer at 1731448359459 (+4 ms)Flushing 1588230740/table: appending metadata at 1731448359475 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731448359475Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6349a5e4: reopening flushed file at 1731448359479 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42578138: reopening flushed file at 1731448359482 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fa7063a: reopening flushed file at 1731448359484 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=11, compaction requested=false at 1731448359487 (+3 ms)Writing region close event to WAL at 1731448359488 (+1 ms)Running coprocessor post-close hooks at 1731448359492 (+4 ms)Closed at 1731448359493 (+1 ms) 2024-11-12T21:52:39,493 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T21:52:39,613 INFO [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(976): stopping server 0c576948786c,33511,1731448356935; all regions closed. 2024-11-12T21:52:39,618 DEBUG [RS:0;0c576948786c:33511 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs 2024-11-12T21:52:39,618 INFO [RS:0;0c576948786c:33511 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0c576948786c%2C33511%2C1731448356935.meta:.meta(num 1731448357593) 2024-11-12T21:52:39,621 DEBUG [RS:0;0c576948786c:33511 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs 2024-11-12T21:52:39,621 INFO [RS:0;0c576948786c:33511 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0c576948786c%2C33511%2C1731448356935:(num 1731448357386) 2024-11-12T21:52:39,621 DEBUG [RS:0;0c576948786c:33511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:39,621 INFO [RS:0;0c576948786c:33511 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T21:52:39,622 INFO [RS:0;0c576948786c:33511 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T21:52:39,622 INFO [RS:0;0c576948786c:33511 {}] hbase.ChoreService(370): Chore service for: regionserver/0c576948786c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T21:52:39,622 INFO [RS:0;0c576948786c:33511 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T21:52:39,622 INFO [regionserver/0c576948786c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T21:52:39,623 INFO [RS:0;0c576948786c:33511 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33511 2024-11-12T21:52:39,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c576948786c,33511,1731448356935 2024-11-12T21:52:39,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T21:52:39,627 INFO [RS:0;0c576948786c:33511 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T21:52:39,629 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c576948786c,33511,1731448356935] 2024-11-12T21:52:39,629 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c576948786c,33511,1731448356935 already deleted, retry=false 2024-11-12T21:52:39,630 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 0c576948786c,33511,1731448356935 on 0c576948786c,41893,1731448356888 2024-11-12T21:52:39,633 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 0c576948786c,33511,1731448356935, splitWal=true, meta=true 2024-11-12T21:52:39,635 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 0c576948786c,33511,1731448356935 (carryingMeta=true) 0c576948786c,33511,1731448356935/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2f048529[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-12T21:52:39,635 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 0c576948786c,33511,1731448356935, splitWal=true, meta=true 2024-11-12T21:52:39,636 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 0c576948786c,33511,1731448356935, splitWal=true, meta=true, isMeta: true 2024-11-12T21:52:39,637 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33511,1731448356935-splitting 2024-11-12T21:52:39,637 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33511,1731448356935-splitting dir is empty, no logs to split. 2024-11-12T21:52:39,637 INFO [PEWorker-1 {}] master.SplitWALManager(105): 0c576948786c,33511,1731448356935 WAL count=0, meta=true 2024-11-12T21:52:39,639 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33511,1731448356935-splitting dir is empty, no logs to split. 2024-11-12T21:52:39,639 INFO [PEWorker-1 {}] master.SplitWALManager(105): 0c576948786c,33511,1731448356935 WAL count=0, meta=true 2024-11-12T21:52:39,639 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 0c576948786c,33511,1731448356935 WAL splitting is done? wals=0, meta=true 2024-11-12T21:52:39,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T21:52:39,641 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T21:52:39,642 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T21:52:39,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:39,729 INFO [RS:0;0c576948786c:33511 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T21:52:39,729 INFO [RS:0;0c576948786c:33511 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c576948786c,33511,1731448356935; zookeeper connection closed. 2024-11-12T21:52:39,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33511-0x10035ec9c760001, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:39,730 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@487a03bb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@487a03bb 2024-11-12T21:52:39,792 DEBUG [0c576948786c:41893 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T21:52:39,794 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0c576948786c,38285,1731448356970 2024-11-12T21:52:39,797 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c576948786c,38285,1731448356970, state=OPENING 2024-11-12T21:52:39,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:39,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:39,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:39,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:39,801 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T21:52:39,801 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0c576948786c,38285,1731448356970}] 2024-11-12T21:52:39,912 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c576948786c:0 server-side Connection retries=45 2024-11-12T21:52:39,912 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:39,913 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:39,913 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T21:52:39,913 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T21:52:39,913 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T21:52:39,913 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T21:52:39,913 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T21:52:39,914 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46045 2024-11-12T21:52:39,915 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46045 connecting to ZooKeeper ensemble=127.0.0.1:54722 2024-11-12T21:52:39,916 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:39,916 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:39,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460450x0, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T21:52:39,919 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46045-0x10035ec9c760003 connected 2024-11-12T21:52:39,919 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-12T21:52:39,919 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-12T21:52:39,920 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T21:52:39,920 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.cache 2024-11-12T21:52:39,921 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T21:52:39,921 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T21:52:39,921 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T21:52:39,921 INFO [Thread-208 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T21:52:39,921 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T21:52:39,921 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T21:52:39,922 INFO [Thread-208 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-12T21:52:39,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T21:52:39,923 INFO [Thread-208 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-12T21:52:39,924 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T21:52:39,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46045 2024-11-12T21:52:39,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46045 2024-11-12T21:52:39,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46045 2024-11-12T21:52:39,926 DEBUG [Thread-208 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-12T21:52:39,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46045 2024-11-12T21:52:39,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46045 2024-11-12T21:52:39,927 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T21:52:39,927 INFO [Thread-208 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-12T21:52:39,927 INFO [Thread-208 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.persistence is complete. 2024-11-12T21:52:39,928 INFO [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(746): ClusterId : 6c64d368-c945-496a-adf0-634c382b4f81 2024-11-12T21:52:39,928 DEBUG [RS:2;0c576948786c:46045 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T21:52:39,931 DEBUG [RS:2;0c576948786c:46045 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T21:52:39,931 DEBUG [RS:2;0c576948786c:46045 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T21:52:39,932 DEBUG [RS:2;0c576948786c:46045 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T21:52:39,933 DEBUG [RS:2;0c576948786c:46045 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ef3bb56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c576948786c/172.17.0.2:0 2024-11-12T21:52:39,945 DEBUG [RS:2;0c576948786c:46045 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;0c576948786c:46045 2024-11-12T21:52:39,946 INFO [RS:2;0c576948786c:46045 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T21:52:39,946 INFO [RS:2;0c576948786c:46045 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T21:52:39,946 DEBUG [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T21:52:39,947 INFO [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c576948786c,41893,1731448356888 with port=46045, startcode=1731448359912 2024-11-12T21:52:39,947 DEBUG [RS:2;0c576948786c:46045 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T21:52:39,949 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38121, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T21:52:39,949 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41893 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c576948786c,46045,1731448359912 2024-11-12T21:52:39,949 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41893 {}] master.ServerManager(517): Registering regionserver=0c576948786c,46045,1731448359912 2024-11-12T21:52:39,951 DEBUG [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a 2024-11-12T21:52:39,951 DEBUG [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T21:52:39,951 DEBUG [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T21:52:39,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T21:52:39,953 DEBUG [RS:2;0c576948786c:46045 {}] zookeeper.ZKUtil(111): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c576948786c,46045,1731448359912 2024-11-12T21:52:39,953 WARN [RS:2;0c576948786c:46045 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T21:52:39,953 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c576948786c,46045,1731448359912] 2024-11-12T21:52:39,953 INFO [RS:2;0c576948786c:46045 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T21:52:39,953 DEBUG [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,46045,1731448359912 2024-11-12T21:52:39,954 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T21:52:39,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49421, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T21:52:39,957 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T21:52:39,957 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T21:52:39,958 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T21:52:39,958 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0c576948786c%2C38285%2C1731448356970.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,38285,1731448356970, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs, maxLogs=55 2024-11-12T21:52:39,958 INFO [RS:2;0c576948786c:46045 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T21:52:39,961 INFO [RS:2;0c576948786c:46045 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T21:52:39,962 INFO [RS:2;0c576948786c:46045 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T21:52:39,962 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:39,962 INFO [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T21:52:39,962 INFO [RS:2;0c576948786c:46045 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T21:52:39,962 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c576948786c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c576948786c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c576948786c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T21:52:39,963 DEBUG [RS:2;0c576948786c:46045 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c576948786c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T21:52:39,969 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:39,969 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:39,969 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:39,969 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:39,969 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:39,969 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,46045,1731448359912-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T21:52:39,976 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,38285,1731448356970/0c576948786c%2C38285%2C1731448356970.meta.1731448359959.meta 2024-11-12T21:52:39,976 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T21:52:39,977 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T21:52:39,977 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T21:52:39,977 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T21:52:39,977 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T21:52:39,977 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T21:52:39,977 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:39,977 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T21:52:39,977 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T21:52:39,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T21:52:39,979 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T21:52:39,979 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:39,980 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/b21dccd548d947c79707d991ac191030, delay=944 ms 2024-11-12T21:52:39,981 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/b21dccd548d947c79707d991ac191030 2024-11-12T21:52:39,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:39,982 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T21:52:39,982 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T21:52:39,982 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:39,984 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/2e69984983754390aab7355bd2268ada, delay=922 ms 2024-11-12T21:52:39,985 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/2e69984983754390aab7355bd2268ada 2024-11-12T21:52:39,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:39,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T21:52:39,986 INFO [RS:2;0c576948786c:46045 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T21:52:39,986 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c576948786c,46045,1731448359912-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:39,986 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:39,986 INFO [RS:2;0c576948786c:46045 {}] regionserver.Replication(171): 0c576948786c,46045,1731448359912 started 2024-11-12T21:52:39,986 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T21:52:39,986 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:39,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:39,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T21:52:39,987 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T21:52:39,987 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:39,989 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/64293acb0f7e48138850a2afe9e4111c, delay=905 ms 2024-11-12T21:52:39,990 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/64293acb0f7e48138850a2afe9e4111c 2024-11-12T21:52:39,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T21:52:39,990 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T21:52:39,991 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740 2024-11-12T21:52:39,991 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740 2024-11-12T21:52:39,991 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T21:52:39,991 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T21:52:39,992 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T21:52:39,992 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T21:52:39,993 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69632080, jitterRate=0.03759884834289551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T21:52:39,993 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T21:52:39,994 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731448359978Writing region info on filesystem at 1731448359978Initializing all the Stores at 1731448359978Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448359978Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448359978Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448359978Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731448359978Cleaning up temporary data from old regions at 1731448359991 (+13 ms)Running coprocessor post-open hooks at 1731448359993 (+2 ms)Region opened successfully at 1731448359994 (+1 ms) 2024-11-12T21:52:39,994 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731448359953 2024-11-12T21:52:39,997 DEBUG [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T21:52:39,997 INFO [RS_OPEN_META-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T21:52:39,997 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=0c576948786c,38285,1731448356970 2024-11-12T21:52:39,998 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c576948786c,38285,1731448356970, state=OPEN 2024-11-12T21:52:39,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:39,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:39,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T21:52:39,999 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0c576948786c,38285,1731448356970 2024-11-12T21:52:39,999 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:39,999 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:39,999 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T21:52:40,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-12T21:52:40,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0c576948786c,38285,1731448356970 in 198 msec 2024-11-12T21:52:40,002 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T21:52:40,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T21:52:40,003 INFO [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(1482): Serving as 0c576948786c,46045,1731448359912, RpcServer on 0c576948786c/172.17.0.2:46045, sessionid=0x10035ec9c760003 2024-11-12T21:52:40,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 361 msec 2024-11-12T21:52:40,003 DEBUG [RS:2;0c576948786c:46045 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T21:52:40,003 DEBUG [RS:2;0c576948786c:46045 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c576948786c,46045,1731448359912 2024-11-12T21:52:40,003 DEBUG [RS:2;0c576948786c:46045 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c576948786c,46045,1731448359912' 2024-11-12T21:52:40,003 DEBUG [RS:2;0c576948786c:46045 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T21:52:40,003 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 0c576948786c,33511,1731448356935 had 2 regions 2024-11-12T21:52:40,004 DEBUG [RS:2;0c576948786c:46045 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T21:52:40,004 DEBUG [RS:2;0c576948786c:46045 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T21:52:40,004 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 0c576948786c,33511,1731448356935, splitWal=true, meta=true, isMeta: false 2024-11-12T21:52:40,004 DEBUG [RS:2;0c576948786c:46045 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T21:52:40,004 DEBUG [RS:2;0c576948786c:46045 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c576948786c,46045,1731448359912 2024-11-12T21:52:40,004 DEBUG [RS:2;0c576948786c:46045 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c576948786c,46045,1731448359912' 2024-11-12T21:52:40,004 DEBUG [RS:2;0c576948786c:46045 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T21:52:40,005 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33511,1731448356935-splitting dir is empty, no logs to split. 2024-11-12T21:52:40,005 INFO [PEWorker-1 {}] master.SplitWALManager(105): 0c576948786c,33511,1731448356935 WAL count=0, meta=false 2024-11-12T21:52:40,005 DEBUG [RS:2;0c576948786c:46045 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T21:52:40,005 DEBUG [RS:2;0c576948786c:46045 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T21:52:40,006 INFO [RS:2;0c576948786c:46045 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T21:52:40,006 INFO [RS:2;0c576948786c:46045 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T21:52:40,006 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,33511,1731448356935-splitting dir is empty, no logs to split. 2024-11-12T21:52:40,006 INFO [PEWorker-1 {}] master.SplitWALManager(105): 0c576948786c,33511,1731448356935 WAL count=0, meta=false 2024-11-12T21:52:40,006 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 0c576948786c,33511,1731448356935 WAL splitting is done? wals=0, meta=false 2024-11-12T21:52:40,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731448357708, region=085504eca23ebcdf3979bbbfc20b190c, ASSIGN}] 2024-11-12T21:52:40,009 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731448357708, region=085504eca23ebcdf3979bbbfc20b190c, ASSIGN 2024-11-12T21:52:40,009 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731448357708, region=085504eca23ebcdf3979bbbfc20b190c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T21:52:40,107 INFO [RS:2;0c576948786c:46045 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T21:52:40,108 INFO [RS:2;0c576948786c:46045 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0c576948786c%2C46045%2C1731448359912, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,46045,1731448359912, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs, maxLogs=55 2024-11-12T21:52:40,129 INFO [RS:2;0c576948786c:46045 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/WALs/0c576948786c,46045,1731448359912/0c576948786c%2C46045%2C1731448359912.1731448360109 2024-11-12T21:52:40,129 DEBUG [RS:2;0c576948786c:46045 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T21:52:40,160 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(204): Hosts are {0c576948786c=0} racks are {/default-rack=0} 2024-11-12T21:52:40,160 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T21:52:40,160 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T21:52:40,160 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T21:52:40,160 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T21:52:40,160 INFO [0c576948786c:41893 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T21:52:40,161 INFO [0c576948786c:41893 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T21:52:40,161 DEBUG [0c576948786c:41893 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T21:52:40,161 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=085504eca23ebcdf3979bbbfc20b190c, regionState=OPENING, regionLocation=0c576948786c,46045,1731448359912 2024-11-12T21:52:40,163 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0c576948786c:33511 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0c576948786c/172.17.0.2:33511 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T21:52:40,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0c576948786c,33511,1731448356935, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0c576948786c,33511,1731448356935, seqNum=-1, error=java.net.ConnectException: Call to address=0c576948786c:33511 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0c576948786c/172.17.0.2:33511 2024-11-12T21:52:40,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0c576948786c,33511,1731448356935, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T21:52:40,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0c576948786c,33511,1731448356935, seqNum=-1 from cache 2024-11-12T21:52:40,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 0c576948786c:33511 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0c576948786c/172.17.0.2:33511 2024-11-12T21:52:40,274 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T21:52:40,274 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c576948786c,38285,1731448356970, seqNum=-1] 2024-11-12T21:52:40,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T21:52:40,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60039, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T21:52:40,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731448357708, region=085504eca23ebcdf3979bbbfc20b190c, ASSIGN because future has completed 2024-11-12T21:52:40,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 085504eca23ebcdf3979bbbfc20b190c, server=0c576948786c,46045,1731448359912}] 2024-11-12T21:52:40,438 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T21:52:40,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34421, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T21:52:40,442 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:40,442 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 085504eca23ebcdf3979bbbfc20b190c, NAME => 'testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c.', STARTKEY => '', ENDKEY => ''} 2024-11-12T21:52:40,443 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731448357708 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,443 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T21:52:40,443 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,443 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,444 INFO [StoreOpener-085504eca23ebcdf3979bbbfc20b190c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,444 INFO [StoreOpener-085504eca23ebcdf3979bbbfc20b190c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 085504eca23ebcdf3979bbbfc20b190c columnFamilyName family 2024-11-12T21:52:40,445 DEBUG [StoreOpener-085504eca23ebcdf3979bbbfc20b190c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T21:52:40,446 DEBUG [StoreFileOpener-085504eca23ebcdf3979bbbfc20b190c-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564, delay=942 ms 2024-11-12T21:52:40,447 DEBUG [StoreOpener-085504eca23ebcdf3979bbbfc20b190c-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564 2024-11-12T21:52:40,447 INFO [StoreOpener-085504eca23ebcdf3979bbbfc20b190c-1 {}] regionserver.HStore(327): Store=085504eca23ebcdf3979bbbfc20b190c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T21:52:40,447 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,448 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,448 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,448 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,449 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,450 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,450 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 085504eca23ebcdf3979bbbfc20b190c; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75441447, jitterRate=0.12416516244411469}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T21:52:40,450 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,451 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 085504eca23ebcdf3979bbbfc20b190c: Running coprocessor pre-open hook at 1731448360443Writing region info on filesystem at 1731448360443Initializing all the Stores at 1731448360443Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731448360443Cleaning up temporary data from old regions at 1731448360449 (+6 ms)Running coprocessor post-open hooks at 1731448360450 (+1 ms)Region opened successfully at 1731448360451 (+1 ms) 2024-11-12T21:52:40,452 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c., pid=11, masterSystemTime=1731448360438 2024-11-12T21:52:40,454 DEBUG [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:40,454 INFO [RS_OPEN_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:40,454 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=085504eca23ebcdf3979bbbfc20b190c, regionState=OPEN, openSeqNum=10, regionLocation=0c576948786c,46045,1731448359912 2024-11-12T21:52:40,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 085504eca23ebcdf3979bbbfc20b190c, server=0c576948786c,46045,1731448359912 because future has completed 2024-11-12T21:52:40,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-12T21:52:40,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 085504eca23ebcdf3979bbbfc20b190c, server=0c576948786c,46045,1731448359912 in 174 msec 2024-11-12T21:52:40,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-12T21:52:40,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731448357708, region=085504eca23ebcdf3979bbbfc20b190c, ASSIGN in 453 msec 2024-11-12T21:52:40,462 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 0c576948786c,33511,1731448356935 after splitting done 2024-11-12T21:52:40,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 0c576948786c,33511,1731448356935, splitWal=true, meta=true in 832 msec 2024-11-12T21:52:40,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T21:52:40,479 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T21:52:40,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T21:52:40,481 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T21:52:40,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T21:52:40,481 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T21:52:40,482 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731448357708 2024-11-12T21:52:40,482 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731448357708 Metrics about Tables on a single HBase RegionServer 2024-11-12T21:52:40,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T21:52:40,503 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T21:52:40,503 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T21:52:40,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:40,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:40,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T21:52:40,504 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T21:52:40,504 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1918435063, stopped=false 2024-11-12T21:52:40,504 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0c576948786c,41893,1731448356888 2024-11-12T21:52:40,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:40,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:40,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T21:52:40,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:40,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:40,505 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T21:52:40,506 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T21:52:40,506 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T21:52:40,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:40,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:40,506 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:40,506 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c576948786c,38285,1731448356970' ***** 2024-11-12T21:52:40,506 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T21:52:40,506 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c576948786c,46045,1731448359912' ***** 2024-11-12T21:52:40,506 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T21:52:40,506 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:40,506 INFO [RS:2;0c576948786c:46045 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T21:52:40,506 INFO [RS:1;0c576948786c:38285 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T21:52:40,507 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T21:52:40,507 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T21:52:40,507 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T21:52:40,507 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T21:52:40,507 INFO [RS:1;0c576948786c:38285 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.persistence 2024-11-12T21:52:40,507 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T21:52:40,507 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T21:52:40,507 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T21:52:40,507 INFO [RS:1;0c576948786c:38285 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T21:52:40,507 INFO [RS:2;0c576948786c:46045 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b59048aa-71f4-6f4e-30b2-bd8a7b004990/bucket.persistence 2024-11-12T21:52:40,508 INFO [RS:2;0c576948786c:46045 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T21:52:40,508 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T21:52:40,508 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T21:52:40,518 DEBUG [RS:1;0c576948786c:38285 {}] bucket.BucketCache(1488): Thread RS:1;0c576948786c:38285 started persisting bucket cache to file 2024-11-12T21:52:40,518 DEBUG [RS:2;0c576948786c:46045 {}] bucket.BucketCache(1488): Thread RS:2;0c576948786c:46045 started persisting bucket cache to file 2024-11-12T21:52:40,518 DEBUG [RS:1;0c576948786c:38285 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T21:52:40,518 DEBUG [RS:2;0c576948786c:46045 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T21:52:40,518 DEBUG [RS:2;0c576948786c:46045 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T21:52:40,518 DEBUG [RS:1;0c576948786c:38285 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T21:52:40,521 DEBUG [RS:1;0c576948786c:38285 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T21:52:40,521 DEBUG [RS:1;0c576948786c:38285 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731448360518 2024-11-12T21:52:40,521 DEBUG [RS:2;0c576948786c:46045 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-12T21:52:40,521 DEBUG [RS:1;0c576948786c:38285 {}] bucket.BucketCache(1510): Thread RS:1;0c576948786c:38285 finished persisting bucket cache to file, renaming 2024-11-12T21:52:40,521 DEBUG [RS:2;0c576948786c:46045 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731448360518 2024-11-12T21:52:40,521 DEBUG [RS:2;0c576948786c:46045 {}] bucket.BucketCache(1510): Thread RS:2;0c576948786c:46045 finished persisting bucket cache to file, renaming 2024-11-12T21:52:40,521 WARN [RS:2;0c576948786c:46045 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-12T21:52:40,521 INFO [RS:2;0c576948786c:46045 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T21:52:40,521 INFO [RS:1;0c576948786c:38285 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T21:52:40,521 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T21:52:40,521 INFO [RS:2;0c576948786c:46045 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T21:52:40,521 INFO [RS:1;0c576948786c:38285 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T21:52:40,521 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T21:52:40,521 INFO [RS:1;0c576948786c:38285 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T21:52:40,521 INFO [RS:2;0c576948786c:46045 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T21:52:40,521 INFO [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(959): stopping server 0c576948786c,38285,1731448356970 2024-11-12T21:52:40,521 INFO [RS:1;0c576948786c:38285 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T21:52:40,521 INFO [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(3091): Received CLOSE for 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,522 INFO [RS:1;0c576948786c:38285 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0c576948786c:38285. 2024-11-12T21:52:40,522 DEBUG [RS:1;0c576948786c:38285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T21:52:40,522 DEBUG [RS:1;0c576948786c:38285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:40,522 INFO [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(959): stopping server 0c576948786c,46045,1731448359912 2024-11-12T21:52:40,522 INFO [RS:1;0c576948786c:38285 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T21:52:40,522 INFO [RS:2;0c576948786c:46045 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T21:52:40,522 INFO [RS:1;0c576948786c:38285 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T21:52:40,522 INFO [RS:1;0c576948786c:38285 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T21:52:40,522 INFO [RS:2;0c576948786c:46045 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;0c576948786c:46045. 2024-11-12T21:52:40,522 INFO [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T21:52:40,522 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T21:52:40,522 DEBUG [RS:2;0c576948786c:46045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T21:52:40,522 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 085504eca23ebcdf3979bbbfc20b190c, disabling compactions & flushes 2024-11-12T21:52:40,522 DEBUG [RS:2;0c576948786c:46045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:40,522 INFO [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T21:52:40,522 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:40,522 DEBUG [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T21:52:40,522 INFO [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T21:52:40,522 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:40,522 DEBUG [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(1325): Online Regions={085504eca23ebcdf3979bbbfc20b190c=testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c.} 2024-11-12T21:52:40,522 DEBUG [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T21:52:40,522 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T21:52:40,522 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. after waiting 0 ms 2024-11-12T21:52:40,522 DEBUG [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(1351): Waiting on 085504eca23ebcdf3979bbbfc20b190c 2024-11-12T21:52:40,522 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:40,522 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T21:52:40,522 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T21:52:40,522 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T21:52:40,523 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T21:52:40,523 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T21:52:40,523 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-12T21:52:40,523 DEBUG [StoreFileCloser-085504eca23ebcdf3979bbbfc20b190c-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/family/de815d09118f4740ac31ed2750c5e564 2024-11-12T21:52:40,530 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/default/testBlockEvictionOnGracefulStop1731448357708/085504eca23ebcdf3979bbbfc20b190c/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-12T21:52:40,530 INFO [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:40,530 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 085504eca23ebcdf3979bbbfc20b190c: Waiting for close lock at 1731448360522Running coprocessor pre-close hooks at 1731448360522Disabling compacts and flushes for region at 1731448360522Disabling writes for close at 1731448360522Writing region close event to WAL at 1731448360524 (+2 ms)Running coprocessor post-close hooks at 1731448360530 (+6 ms)Closed at 1731448360530 2024-11-12T21:52:40,530 DEBUG [RS_CLOSE_REGION-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c. 2024-11-12T21:52:40,534 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1d1545106e7b462894ddf483e4498527 2024-11-12T21:52:40,548 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/info/1d1545106e7b462894ddf483e4498527 is 209, key is testBlockEvictionOnGracefulStop1731448357708,,1731448357714.085504eca23ebcdf3979bbbfc20b190c./info:regioninfo/1731448360454/Put/seqid=0 2024-11-12T21:52:40,549 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/info/1d1545106e7b462894ddf483e4498527 2024-11-12T21:52:40,551 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/.tmp/info/1d1545106e7b462894ddf483e4498527 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/1d1545106e7b462894ddf483e4498527 2024-11-12T21:52:40,552 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/1d1545106e7b462894ddf483e4498527, delay=980 ms 2024-11-12T21:52:40,554 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/1d1545106e7b462894ddf483e4498527, entries=8, sequenceid=19, filesize=6.6 K 2024-11-12T21:52:40,554 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false 2024-11-12T21:52:40,555 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/b21dccd548d947c79707d991ac191030 2024-11-12T21:52:40,555 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/info/1d1545106e7b462894ddf483e4498527 2024-11-12T21:52:40,556 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/ns/2e69984983754390aab7355bd2268ada 2024-11-12T21:52:40,556 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/table/64293acb0f7e48138850a2afe9e4111c 2024-11-12T21:52:40,558 INFO [regionserver/0c576948786c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T21:52:40,560 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-12T21:52:40,561 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T21:52:40,561 INFO [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T21:52:40,561 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731448360522Running coprocessor pre-close hooks at 1731448360522Disabling compacts and flushes for region at 1731448360522Disabling writes for close at 1731448360523 (+1 ms)Obtaining lock to block concurrent updates at 1731448360523Preparing flush snapshotting stores in 1588230740 at 1731448360523Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731448360523Flushing stores of hbase:meta,,1.1588230740 at 1731448360523Flushing 1588230740/info: creating writer at 1731448360524 (+1 ms)Flushing 1588230740/info: appending metadata at 1731448360547 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731448360547Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a6cb0de: reopening flushed file at 1731448360551 (+4 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false at 1731448360554 (+3 ms)Writing region close event to WAL at 1731448360557 (+3 ms)Running coprocessor post-close hooks at 1731448360561 (+4 ms)Closed at 1731448360561 2024-11-12T21:52:40,562 DEBUG [RS_CLOSE_META-regionserver/0c576948786c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T21:52:40,570 INFO [regionserver/0c576948786c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T21:52:40,722 INFO [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(976): stopping server 0c576948786c,46045,1731448359912; all regions closed. 2024-11-12T21:52:40,722 INFO [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(976): stopping server 0c576948786c,38285,1731448356970; all regions closed. 2024-11-12T21:52:40,726 DEBUG [RS:2;0c576948786c:46045 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs 2024-11-12T21:52:40,726 INFO [RS:2;0c576948786c:46045 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0c576948786c%2C46045%2C1731448359912:(num 1731448360109) 2024-11-12T21:52:40,726 DEBUG [RS:2;0c576948786c:46045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:40,726 DEBUG [RS:1;0c576948786c:38285 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs 2024-11-12T21:52:40,726 INFO [RS:2;0c576948786c:46045 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T21:52:40,726 INFO [RS:1;0c576948786c:38285 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0c576948786c%2C38285%2C1731448356970.meta:.meta(num 1731448359959) 2024-11-12T21:52:40,726 INFO [RS:2;0c576948786c:46045 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T21:52:40,727 INFO [RS:2;0c576948786c:46045 {}] hbase.ChoreService(370): Chore service for: regionserver/0c576948786c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T21:52:40,727 INFO [RS:2;0c576948786c:46045 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T21:52:40,727 INFO [RS:2;0c576948786c:46045 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T21:52:40,727 INFO [regionserver/0c576948786c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T21:52:40,727 INFO [RS:2;0c576948786c:46045 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T21:52:40,727 INFO [RS:2;0c576948786c:46045 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T21:52:40,728 INFO [RS:2;0c576948786c:46045 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46045 2024-11-12T21:52:40,730 DEBUG [RS:1;0c576948786c:38285 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/oldWALs 2024-11-12T21:52:40,730 INFO [RS:1;0c576948786c:38285 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0c576948786c%2C38285%2C1731448356970:(num 1731448357393) 2024-11-12T21:52:40,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c576948786c,46045,1731448359912 2024-11-12T21:52:40,730 DEBUG [RS:1;0c576948786c:38285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T21:52:40,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T21:52:40,730 INFO [RS:1;0c576948786c:38285 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T21:52:40,730 INFO [RS:2;0c576948786c:46045 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T21:52:40,730 INFO [RS:1;0c576948786c:38285 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T21:52:40,731 INFO [RS:1;0c576948786c:38285 {}] hbase.ChoreService(370): Chore service for: regionserver/0c576948786c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T21:52:40,731 INFO [RS:1;0c576948786c:38285 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T21:52:40,731 INFO [regionserver/0c576948786c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T21:52:40,731 INFO [RS:1;0c576948786c:38285 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38285 2024-11-12T21:52:40,732 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c576948786c,46045,1731448359912] 2024-11-12T21:52:40,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c576948786c,38285,1731448356970 2024-11-12T21:52:40,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T21:52:40,733 INFO [RS:1;0c576948786c:38285 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T21:52:40,733 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c576948786c,46045,1731448359912 already deleted, retry=false 2024-11-12T21:52:40,734 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c576948786c,46045,1731448359912 expired; onlineServers=1 2024-11-12T21:52:40,734 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c576948786c,38285,1731448356970] 2024-11-12T21:52:40,735 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c576948786c,38285,1731448356970 already deleted, retry=false 2024-11-12T21:52:40,735 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c576948786c,38285,1731448356970 expired; onlineServers=0 2024-11-12T21:52:40,735 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0c576948786c,41893,1731448356888' ***** 2024-11-12T21:52:40,735 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T21:52:40,735 INFO [M:0;0c576948786c:41893 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T21:52:40,735 INFO [M:0;0c576948786c:41893 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T21:52:40,735 DEBUG [M:0;0c576948786c:41893 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T21:52:40,735 DEBUG [M:0;0c576948786c:41893 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T21:52:40,735 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T21:52:40,735 DEBUG [master/0c576948786c:0:becomeActiveMaster-HFileCleaner.large.0-1731448357193 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c576948786c:0:becomeActiveMaster-HFileCleaner.large.0-1731448357193,5,FailOnTimeoutGroup] 2024-11-12T21:52:40,735 DEBUG [master/0c576948786c:0:becomeActiveMaster-HFileCleaner.small.0-1731448357195 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c576948786c:0:becomeActiveMaster-HFileCleaner.small.0-1731448357195,5,FailOnTimeoutGroup] 2024-11-12T21:52:40,736 INFO [M:0;0c576948786c:41893 {}] hbase.ChoreService(370): Chore service for: master/0c576948786c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T21:52:40,736 INFO [M:0;0c576948786c:41893 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T21:52:40,736 DEBUG [M:0;0c576948786c:41893 {}] master.HMaster(1795): Stopping service threads 2024-11-12T21:52:40,736 INFO [M:0;0c576948786c:41893 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T21:52:40,736 INFO [M:0;0c576948786c:41893 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T21:52:40,736 INFO [M:0;0c576948786c:41893 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T21:52:40,736 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T21:52:40,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T21:52:40,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T21:52:40,737 DEBUG [M:0;0c576948786c:41893 {}] zookeeper.ZKUtil(347): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T21:52:40,737 WARN [M:0;0c576948786c:41893 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T21:52:40,737 INFO [M:0;0c576948786c:41893 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/.lastflushedseqids 2024-11-12T21:52:40,740 INFO [M:0;0c576948786c:41893 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T21:52:40,740 INFO [M:0;0c576948786c:41893 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T21:52:40,740 DEBUG [M:0;0c576948786c:41893 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T21:52:40,740 INFO [M:0;0c576948786c:41893 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:40,740 DEBUG [M:0;0c576948786c:41893 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:40,740 DEBUG [M:0;0c576948786c:41893 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T21:52:40,740 DEBUG [M:0;0c576948786c:41893 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:40,740 INFO [M:0;0c576948786c:41893 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-12T21:52:40,748 DEBUG [M:0;0c576948786c:41893 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1b9f899fb313448fb03c814bd27a54c9 2024-11-12T21:52:40,762 DEBUG [M:0;0c576948786c:41893 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b9f899fb313448fb03c814bd27a54c9 is 82, key is hbase:meta,,1/info:regioninfo/1731448359997/Put/seqid=0 2024-11-12T21:52:40,762 INFO [M:0;0c576948786c:41893 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b9f899fb313448fb03c814bd27a54c9 2024-11-12T21:52:40,769 DEBUG [M:0;0c576948786c:41893 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on eb1cf379dbe94ce494b9eb51d2751a57 2024-11-12T21:52:40,782 DEBUG [M:0;0c576948786c:41893 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eb1cf379dbe94ce494b9eb51d2751a57 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731448358087/Put/seqid=0 2024-11-12T21:52:40,783 INFO [M:0;0c576948786c:41893 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eb1cf379dbe94ce494b9eb51d2751a57 2024-11-12T21:52:40,791 DEBUG [M:0;0c576948786c:41893 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 81d100c7d9d241d4b92678072abec3c7 2024-11-12T21:52:40,805 DEBUG [M:0;0c576948786c:41893 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/81d100c7d9d241d4b92678072abec3c7 is 69, key is 0c576948786c,38285,1731448356970/rs:state/1731448357227/Put/seqid=0 2024-11-12T21:52:40,805 INFO [M:0;0c576948786c:41893 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/81d100c7d9d241d4b92678072abec3c7 2024-11-12T21:52:40,807 INFO [M:0;0c576948786c:41893 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 81d100c7d9d241d4b92678072abec3c7 2024-11-12T21:52:40,807 DEBUG [M:0;0c576948786c:41893 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b9f899fb313448fb03c814bd27a54c9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1b9f899fb313448fb03c814bd27a54c9 2024-11-12T21:52:40,809 INFO [M:0;0c576948786c:41893 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1b9f899fb313448fb03c814bd27a54c9, entries=14, sequenceid=102, filesize=5.9 K 2024-11-12T21:52:40,809 DEBUG [M:0;0c576948786c:41893 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eb1cf379dbe94ce494b9eb51d2751a57 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eb1cf379dbe94ce494b9eb51d2751a57 2024-11-12T21:52:40,811 INFO [M:0;0c576948786c:41893 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eb1cf379dbe94ce494b9eb51d2751a57, entries=11, sequenceid=102, filesize=6.5 K 2024-11-12T21:52:40,811 DEBUG [M:0;0c576948786c:41893 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/81d100c7d9d241d4b92678072abec3c7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/81d100c7d9d241d4b92678072abec3c7 2024-11-12T21:52:40,812 INFO [M:0;0c576948786c:41893 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 81d100c7d9d241d4b92678072abec3c7 2024-11-12T21:52:40,813 INFO [M:0;0c576948786c:41893 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c1d492c4-bb33-5093-f136-3c1990ce64b9/ddd367ce-99b1-e526-ceda-13ba19bb704a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/81d100c7d9d241d4b92678072abec3c7, entries=3, sequenceid=102, filesize=5.3 K 2024-11-12T21:52:40,813 INFO [M:0;0c576948786c:41893 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=102, compaction requested=false 2024-11-12T21:52:40,815 INFO [M:0;0c576948786c:41893 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T21:52:40,815 DEBUG [M:0;0c576948786c:41893 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731448360740Disabling compacts and flushes for region at 1731448360740Disabling writes for close at 1731448360740Obtaining lock to block concurrent updates at 1731448360740Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731448360740Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1731448360741 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731448360741Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731448360741Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731448360761 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731448360761Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731448360764 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731448360782 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731448360782Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731448360785 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731448360804 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731448360804Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e51d20c: reopening flushed file at 1731448360807 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c30b1e4: reopening flushed file at 1731448360809 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36016d0e: reopening flushed file at 1731448360811 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=102, compaction requested=false at 1731448360813 (+2 ms)Writing region close event to WAL at 1731448360815 (+2 ms)Closed at 1731448360815 2024-11-12T21:52:40,816 INFO [M:0;0c576948786c:41893 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T21:52:40,816 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T21:52:40,817 INFO [M:0;0c576948786c:41893 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41893 2024-11-12T21:52:40,817 INFO [M:0;0c576948786c:41893 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T21:52:40,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:40,833 INFO [RS:2;0c576948786c:46045 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T21:52:40,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46045-0x10035ec9c760003, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:40,833 INFO [RS:2;0c576948786c:46045 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c576948786c,46045,1731448359912; zookeeper connection closed. 2024-11-12T21:52:40,833 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c50daff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c50daff 2024-11-12T21:52:40,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:40,834 INFO [RS:1;0c576948786c:38285 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T21:52:40,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38285-0x10035ec9c760002, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:40,834 INFO [RS:1;0c576948786c:38285 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c576948786c,38285,1731448356970; zookeeper connection closed. 2024-11-12T21:52:40,835 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ae98df3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ae98df3 2024-11-12T21:52:40,835 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-12T21:52:40,918 INFO [M:0;0c576948786c:41893 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T21:52:40,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:40,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41893-0x10035ec9c760000, quorum=127.0.0.1:54722, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T21:52:40,921 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T21:52:40,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T21:52:40,923 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T21:52:40,931 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=370 (was 370), ProcessCount=11 (was 11), AvailableMemoryMB=9356 (was 9449)