2024-11-22 08:55:19,063 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-22 08:55:19,080 main DEBUG Took 0.014878 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 08:55:19,080 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 08:55:19,081 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 08:55:19,081 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 08:55:19,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,088 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 08:55:19,101 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,102 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,103 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,104 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,104 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,105 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,106 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,107 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,108 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,109 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,109 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,110 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,110 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,111 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,111 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,112 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,113 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,113 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,114 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,114 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,115 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,115 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,116 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 08:55:19,116 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,117 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 08:55:19,119 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 08:55:19,120 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 08:55:19,122 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 08:55:19,123 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 08:55:19,124 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 08:55:19,125 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 08:55:19,136 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 08:55:19,139 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 08:55:19,141 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 08:55:19,142 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 08:55:19,142 main DEBUG createAppenders(={Console}) 2024-11-22 08:55:19,143 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-22 08:55:19,144 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-22 08:55:19,144 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-22 08:55:19,145 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 08:55:19,145 main DEBUG OutputStream closed 2024-11-22 08:55:19,146 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 08:55:19,146 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 08:55:19,146 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-22 08:55:19,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 08:55:19,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 08:55:19,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 08:55:19,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 08:55:19,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 08:55:19,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 08:55:19,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 08:55:19,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 08:55:19,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 08:55:19,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 08:55:19,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 08:55:19,257 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 08:55:19,257 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 08:55:19,258 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 08:55:19,258 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 08:55:19,259 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 08:55:19,259 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 08:55:19,260 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 08:55:19,264 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 08:55:19,264 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-22 08:55:19,265 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 08:55:19,266 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-22T08:55:19,283 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-22 08:55:19,287 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 08:55:19,288 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T08:55:19,684 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295 2024-11-22T08:55:19,736 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=313, ProcessCount=11, AvailableMemoryMB=5871 2024-11-22T08:55:19,887 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T08:55:20,033 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:20,063 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/cluster_64a52d69-c9b8-a8bd-0477-ed0694f403fe, deleteOnExit=true 2024-11-22T08:55:20,198 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/cluster_64a52d69-c9b8-a8bd-0477-ed0694f403fe/zookeeper_0, clientPort=52585, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/cluster_64a52d69-c9b8-a8bd-0477-ed0694f403fe/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/cluster_64a52d69-c9b8-a8bd-0477-ed0694f403fe/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T08:55:20,231 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52585 2024-11-22T08:55:20,232 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:20,397 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a with version=8 2024-11-22T08:55:20,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/hbase-staging 2024-11-22T08:55:20,572 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T08:55:20,925 INFO [Time-limited test {}] client.ConnectionUtils(128): master/296fbbacac79:0 server-side Connection retries=45 2024-11-22T08:55:20,939 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:20,941 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:20,967 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T08:55:20,967 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:20,968 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T08:55:21,169 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T08:55:21,327 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T08:55:21,430 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T08:55:21,456 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T08:55:21,462 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T08:55:21,520 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 92676 (auto-detected) 2024-11-22T08:55:21,522 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T08:55:21,576 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40087 2024-11-22T08:55:21,616 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40087 connecting to ZooKeeper ensemble=127.0.0.1:52585 2024-11-22T08:55:21,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400870x0, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T08:55:21,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40087-0x10066a495d70000 connected 2024-11-22T08:55:21,730 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:21,731 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:21,743 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:21,748 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a, hbase.cluster.distributed=false 2024-11-22T08:55:21,782 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T08:55:21,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40087 2024-11-22T08:55:21,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40087 2024-11-22T08:55:21,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40087 2024-11-22T08:55:21,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40087 2024-11-22T08:55:21,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40087 2024-11-22T08:55:21,910 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/296fbbacac79:0 server-side Connection retries=45 2024-11-22T08:55:21,912 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:21,912 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:21,913 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T08:55:21,913 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:21,914 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T08:55:21,918 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T08:55:21,922 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T08:55:21,923 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45617 2024-11-22T08:55:21,927 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45617 connecting to ZooKeeper ensemble=127.0.0.1:52585 2024-11-22T08:55:21,928 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:21,931 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:21,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456170x0, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T08:55:21,945 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:456170x0, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:21,945 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45617-0x10066a495d70001 connected 2024-11-22T08:55:21,953 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T08:55:21,976 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/bucket.cache 2024-11-22T08:55:21,986 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T08:55:21,988 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T08:55:21,991 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T08:55:22,003 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T08:55:22,006 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T08:55:22,020 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T08:55:22,025 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T08:55:22,026 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/bucket.persistence is complete. 2024-11-22T08:55:22,028 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T08:55:22,033 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T08:55:22,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T08:55:22,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45617 2024-11-22T08:55:22,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45617 2024-11-22T08:55:22,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45617 2024-11-22T08:55:22,081 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45617 2024-11-22T08:55:22,084 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45617 2024-11-22T08:55:22,115 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/296fbbacac79:0 server-side Connection retries=45 2024-11-22T08:55:22,115 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:22,118 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:22,118 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T08:55:22,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:22,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T08:55:22,122 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T08:55:22,126 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T08:55:22,130 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34699 2024-11-22T08:55:22,135 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34699 connecting to ZooKeeper ensemble=127.0.0.1:52585 2024-11-22T08:55:22,136 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:22,141 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:22,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:346990x0, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T08:55:22,161 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:346990x0, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:22,162 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T08:55:22,165 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34699-0x10066a495d70002 connected 2024-11-22T08:55:22,166 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/bucket.cache 2024-11-22T08:55:22,166 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T08:55:22,166 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T08:55:22,166 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T08:55:22,170 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T08:55:22,171 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T08:55:22,172 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T08:55:22,172 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/bucket.persistence is complete. 2024-11-22T08:55:22,177 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T08:55:22,181 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T08:55:22,183 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T08:55:22,187 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T08:55:22,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34699 2024-11-22T08:55:22,197 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34699 2024-11-22T08:55:22,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34699 2024-11-22T08:55:22,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34699 2024-11-22T08:55:22,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34699 2024-11-22T08:55:22,231 DEBUG [M:0;296fbbacac79:40087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;296fbbacac79:40087 2024-11-22T08:55:22,232 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/296fbbacac79,40087,1732265720689 2024-11-22T08:55:22,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:22,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:22,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:22,246 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/296fbbacac79,40087,1732265720689 2024-11-22T08:55:22,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T08:55:22,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T08:55:22,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:22,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:22,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:22,269 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T08:55:22,270 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/296fbbacac79,40087,1732265720689 from backup master directory 2024-11-22T08:55:22,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/296fbbacac79,40087,1732265720689 2024-11-22T08:55:22,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:22,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:22,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:22,275 WARN [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T08:55:22,275 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=296fbbacac79,40087,1732265720689 2024-11-22T08:55:22,278 INFO [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T08:55:22,281 INFO [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T08:55:22,291 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/hbase.id] with ID: 48e01fbb-7ffd-4ac0-8c36-7c687543214c 2024-11-22T08:55:22,291 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/.tmp/hbase.id 2024-11-22T08:55:22,297 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/hbase.id] 2024-11-22T08:55:22,352 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:22,359 INFO [master/296fbbacac79:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T08:55:22,396 INFO [master/296fbbacac79:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-22T08:55:22,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:22,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:22,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:22,438 INFO [master/296fbbacac79:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T08:55:22,442 INFO [master/296fbbacac79:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T08:55:22,488 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T08:55:22,494 INFO [master/296fbbacac79:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T08:55:22,549 INFO [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store 2024-11-22T08:55:22,568 INFO [master/296fbbacac79:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T08:55:22,572 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:22,573 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T08:55:22,573 INFO [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:22,573 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:22,575 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T08:55:22,575 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:22,575 INFO [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:22,576 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732265722573Disabling compacts and flushes for region at 1732265722573Disabling writes for close at 1732265722575 (+2 ms)Writing region close event to WAL at 1732265722575Closed at 1732265722575 2024-11-22T08:55:22,576 WARN [master/296fbbacac79:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/.initializing 2024-11-22T08:55:22,577 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/WALs/296fbbacac79,40087,1732265720689 2024-11-22T08:55:22,592 INFO [master/296fbbacac79:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T08:55:22,615 INFO [master/296fbbacac79:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=296fbbacac79%2C40087%2C1732265720689, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/WALs/296fbbacac79,40087,1732265720689, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/oldWALs, maxLogs=10 2024-11-22T08:55:22,666 INFO [master/296fbbacac79:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/WALs/296fbbacac79,40087,1732265720689/296fbbacac79%2C40087%2C1732265720689.1732265722621 2024-11-22T08:55:22,667 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T08:55:22,668 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T08:55:22,669 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:22,674 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,676 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T08:55:22,757 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:22,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:22,760 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,775 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T08:55:22,775 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:22,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:22,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,787 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T08:55:22,788 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:22,788 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:22,789 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,799 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T08:55:22,799 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:22,799 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:22,800 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,805 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,806 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,810 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,811 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,818 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T08:55:22,823 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:22,832 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T08:55:22,833 INFO [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72051311, jitterRate=0.07364819943904877}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T08:55:22,842 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732265722692Initializing all the Stores at 1732265722693 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265722694 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265722695 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265722695Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265722695Cleaning up temporary data from old regions at 1732265722811 (+116 ms)Region opened successfully at 1732265722842 (+31 ms) 2024-11-22T08:55:22,843 INFO [master/296fbbacac79:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T08:55:22,885 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbbf432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=296fbbacac79/172.17.0.2:0 2024-11-22T08:55:22,925 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T08:55:22,938 INFO [master/296fbbacac79:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T08:55:22,938 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T08:55:22,942 INFO [master/296fbbacac79:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T08:55:22,942 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T08:55:22,949 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T08:55:22,949 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T08:55:22,978 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T08:55:22,990 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T08:55:22,992 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T08:55:22,996 INFO [master/296fbbacac79:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T08:55:22,999 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T08:55:23,001 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T08:55:23,004 INFO [master/296fbbacac79:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T08:55:23,009 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T08:55:23,011 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T08:55:23,013 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T08:55:23,015 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T08:55:23,048 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T08:55:23,057 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T08:55:23,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:23,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:23,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:23,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,080 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=296fbbacac79,40087,1732265720689, sessionid=0x10066a495d70000, setting cluster-up flag (Was=false) 2024-11-22T08:55:23,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,098 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T08:55:23,101 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=296fbbacac79,40087,1732265720689 2024-11-22T08:55:23,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,108 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T08:55:23,110 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=296fbbacac79,40087,1732265720689 2024-11-22T08:55:23,115 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T08:55:23,190 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T08:55:23,202 INFO [master/296fbbacac79:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T08:55:23,210 INFO [master/296fbbacac79:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T08:55:23,212 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(746): ClusterId : 48e01fbb-7ffd-4ac0-8c36-7c687543214c 2024-11-22T08:55:23,212 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(746): ClusterId : 48e01fbb-7ffd-4ac0-8c36-7c687543214c 2024-11-22T08:55:23,215 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T08:55:23,215 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T08:55:23,217 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 296fbbacac79,40087,1732265720689 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T08:55:23,222 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T08:55:23,223 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T08:55:23,223 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T08:55:23,223 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T08:55:23,227 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T08:55:23,227 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T08:55:23,228 DEBUG [RS:1;296fbbacac79:34699 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b668fae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=296fbbacac79/172.17.0.2:0 2024-11-22T08:55:23,228 DEBUG [RS:0;296fbbacac79:45617 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f7b11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=296fbbacac79/172.17.0.2:0 2024-11-22T08:55:23,229 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/296fbbacac79:0, corePoolSize=5, maxPoolSize=5 2024-11-22T08:55:23,229 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/296fbbacac79:0, corePoolSize=5, maxPoolSize=5 2024-11-22T08:55:23,230 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/296fbbacac79:0, corePoolSize=5, maxPoolSize=5 2024-11-22T08:55:23,230 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/296fbbacac79:0, corePoolSize=5, maxPoolSize=5 2024-11-22T08:55:23,230 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/296fbbacac79:0, corePoolSize=10, maxPoolSize=10 2024-11-22T08:55:23,230 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,231 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/296fbbacac79:0, corePoolSize=2, maxPoolSize=2 2024-11-22T08:55:23,231 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,235 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732265753235 2024-11-22T08:55:23,237 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T08:55:23,238 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T08:55:23,238 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T08:55:23,241 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T08:55:23,243 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:23,244 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T08:55:23,246 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T08:55:23,247 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T08:55:23,247 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T08:55:23,248 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T08:55:23,253 DEBUG [RS:1;296fbbacac79:34699 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;296fbbacac79:34699 2024-11-22T08:55:23,254 DEBUG [RS:0;296fbbacac79:45617 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;296fbbacac79:45617 2024-11-22T08:55:23,254 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,258 INFO [RS:0;296fbbacac79:45617 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T08:55:23,258 INFO [RS:1;296fbbacac79:34699 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T08:55:23,258 INFO [RS:0;296fbbacac79:45617 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T08:55:23,258 INFO [RS:1;296fbbacac79:34699 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T08:55:23,258 DEBUG [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T08:55:23,258 DEBUG [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T08:55:23,260 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T08:55:23,261 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T08:55:23,262 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(2659): reportForDuty to master=296fbbacac79,40087,1732265720689 with port=34699, startcode=1732265722113 2024-11-22T08:55:23,262 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T08:55:23,262 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(2659): reportForDuty to master=296fbbacac79,40087,1732265720689 with port=45617, startcode=1732265721863 2024-11-22T08:55:23,264 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T08:55:23,265 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T08:55:23,267 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.large.0-1732265723266,5,FailOnTimeoutGroup] 2024-11-22T08:55:23,269 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.small.0-1732265723267,5,FailOnTimeoutGroup] 2024-11-22T08:55:23,269 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,270 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T08:55:23,271 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,271 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,275 DEBUG [RS:1;296fbbacac79:34699 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T08:55:23,275 DEBUG [RS:0;296fbbacac79:45617 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T08:55:23,284 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T08:55:23,285 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a 2024-11-22T08:55:23,291 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:23,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T08:55:23,297 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T08:55:23,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:23,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:23,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T08:55:23,302 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T08:55:23,302 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:23,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:23,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T08:55:23,308 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T08:55:23,308 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:23,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:23,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T08:55:23,316 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T08:55:23,316 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:23,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:23,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T08:55:23,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740 2024-11-22T08:55:23,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740 2024-11-22T08:55:23,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T08:55:23,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T08:55:23,324 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T08:55:23,325 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T08:55:23,336 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T08:55:23,337 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66991944, jitterRate=-0.001742243766784668}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T08:55:23,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732265723292Initializing all the Stores at 1732265723292Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265723292Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265723292Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265723292Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265723292Cleaning up temporary data from old regions at 1732265723324 (+32 ms)Region opened successfully at 1732265723343 (+19 ms) 2024-11-22T08:55:23,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T08:55:23,344 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T08:55:23,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T08:55:23,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T08:55:23,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T08:55:23,346 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T08:55:23,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732265723343Disabling compacts and flushes for region at 1732265723343Disabling writes for close at 1732265723344 (+1 ms)Writing region close event to WAL at 1732265723345 (+1 ms)Closed at 1732265723346 (+1 ms) 2024-11-22T08:55:23,350 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T08:55:23,350 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47441, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T08:55:23,350 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33561, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T08:55:23,351 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T08:55:23,359 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40087 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 296fbbacac79,34699,1732265722113 2024-11-22T08:55:23,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T08:55:23,362 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40087 {}] master.ServerManager(517): Registering regionserver=296fbbacac79,34699,1732265722113 2024-11-22T08:55:23,369 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T08:55:23,371 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40087 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 296fbbacac79,45617,1732265721863 2024-11-22T08:55:23,371 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40087 {}] master.ServerManager(517): Registering regionserver=296fbbacac79,45617,1732265721863 2024-11-22T08:55:23,371 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T08:55:23,375 DEBUG [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a 2024-11-22T08:55:23,375 DEBUG [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a 2024-11-22T08:55:23,375 DEBUG [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T08:55:23,375 DEBUG [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T08:55:23,375 DEBUG [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T08:55:23,375 DEBUG [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T08:55:23,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T08:55:23,381 DEBUG [RS:0;296fbbacac79:45617 {}] zookeeper.ZKUtil(111): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/296fbbacac79,45617,1732265721863 2024-11-22T08:55:23,381 WARN [RS:0;296fbbacac79:45617 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T08:55:23,381 INFO [RS:0;296fbbacac79:45617 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T08:55:23,381 DEBUG [RS:1;296fbbacac79:34699 {}] zookeeper.ZKUtil(111): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/296fbbacac79,34699,1732265722113 2024-11-22T08:55:23,382 DEBUG [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,45617,1732265721863 2024-11-22T08:55:23,382 WARN [RS:1;296fbbacac79:34699 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T08:55:23,382 INFO [RS:1;296fbbacac79:34699 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T08:55:23,382 DEBUG [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,34699,1732265722113 2024-11-22T08:55:23,384 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [296fbbacac79,34699,1732265722113] 2024-11-22T08:55:23,384 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [296fbbacac79,45617,1732265721863] 2024-11-22T08:55:23,409 INFO [RS:1;296fbbacac79:34699 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T08:55:23,409 INFO [RS:0;296fbbacac79:45617 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T08:55:23,430 INFO [RS:1;296fbbacac79:34699 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T08:55:23,430 INFO [RS:0;296fbbacac79:45617 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T08:55:23,435 INFO [RS:1;296fbbacac79:34699 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T08:55:23,435 INFO [RS:0;296fbbacac79:45617 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T08:55:23,435 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,435 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,436 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T08:55:23,437 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T08:55:23,441 INFO [RS:1;296fbbacac79:34699 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T08:55:23,441 INFO [RS:0;296fbbacac79:45617 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T08:55:23,443 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,443 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,444 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,444 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,444 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,444 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,444 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,444 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,444 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,444 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,444 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,444 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,445 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/296fbbacac79:0, corePoolSize=2, maxPoolSize=2 2024-11-22T08:55:23,445 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/296fbbacac79:0, corePoolSize=2, maxPoolSize=2 2024-11-22T08:55:23,445 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,445 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,445 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,445 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,445 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,445 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,445 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,445 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,445 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,446 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,446 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,446 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:23,446 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/296fbbacac79:0, corePoolSize=3, maxPoolSize=3 2024-11-22T08:55:23,446 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/296fbbacac79:0, corePoolSize=3, maxPoolSize=3 2024-11-22T08:55:23,446 DEBUG [RS:1;296fbbacac79:34699 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/296fbbacac79:0, corePoolSize=3, maxPoolSize=3 2024-11-22T08:55:23,446 DEBUG [RS:0;296fbbacac79:45617 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/296fbbacac79:0, corePoolSize=3, maxPoolSize=3 2024-11-22T08:55:23,447 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,447 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,447 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,447 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,448 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,448 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,448 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,448 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,448 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,448 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,448 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,45617,1732265721863-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T08:55:23,448 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,34699,1732265722113-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T08:55:23,468 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T08:55:23,470 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,45617,1732265721863-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,471 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,471 INFO [RS:0;296fbbacac79:45617 {}] regionserver.Replication(171): 296fbbacac79,45617,1732265721863 started 2024-11-22T08:55:23,473 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T08:55:23,473 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,34699,1732265722113-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,474 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,474 INFO [RS:1;296fbbacac79:34699 {}] regionserver.Replication(171): 296fbbacac79,34699,1732265722113 started 2024-11-22T08:55:23,489 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,490 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(1482): Serving as 296fbbacac79,45617,1732265721863, RpcServer on 296fbbacac79/172.17.0.2:45617, sessionid=0x10066a495d70001 2024-11-22T08:55:23,491 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T08:55:23,491 DEBUG [RS:0;296fbbacac79:45617 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 296fbbacac79,45617,1732265721863 2024-11-22T08:55:23,491 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '296fbbacac79,45617,1732265721863' 2024-11-22T08:55:23,492 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T08:55:23,493 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T08:55:23,494 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T08:55:23,494 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T08:55:23,494 DEBUG [RS:0;296fbbacac79:45617 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 296fbbacac79,45617,1732265721863 2024-11-22T08:55:23,494 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '296fbbacac79,45617,1732265721863' 2024-11-22T08:55:23,494 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T08:55:23,495 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T08:55:23,496 DEBUG [RS:0;296fbbacac79:45617 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T08:55:23,496 INFO [RS:0;296fbbacac79:45617 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T08:55:23,496 INFO [RS:0;296fbbacac79:45617 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T08:55:23,497 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:23,498 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(1482): Serving as 296fbbacac79,34699,1732265722113, RpcServer on 296fbbacac79/172.17.0.2:34699, sessionid=0x10066a495d70002 2024-11-22T08:55:23,498 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T08:55:23,498 DEBUG [RS:1;296fbbacac79:34699 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 296fbbacac79,34699,1732265722113 2024-11-22T08:55:23,498 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '296fbbacac79,34699,1732265722113' 2024-11-22T08:55:23,498 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T08:55:23,499 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T08:55:23,500 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T08:55:23,500 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T08:55:23,500 DEBUG [RS:1;296fbbacac79:34699 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 296fbbacac79,34699,1732265722113 2024-11-22T08:55:23,500 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '296fbbacac79,34699,1732265722113' 2024-11-22T08:55:23,500 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T08:55:23,501 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T08:55:23,502 DEBUG [RS:1;296fbbacac79:34699 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T08:55:23,502 INFO [RS:1;296fbbacac79:34699 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T08:55:23,502 INFO [RS:1;296fbbacac79:34699 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T08:55:23,522 WARN [296fbbacac79:40087 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T08:55:23,603 INFO [RS:0;296fbbacac79:45617 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T08:55:23,603 INFO [RS:1;296fbbacac79:34699 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T08:55:23,604 INFO [RS:0;296fbbacac79:45617 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=296fbbacac79%2C45617%2C1732265721863, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,45617,1732265721863, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs, maxLogs=55 2024-11-22T08:55:23,605 INFO [RS:1;296fbbacac79:34699 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=296fbbacac79%2C34699%2C1732265722113, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,34699,1732265722113, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs, maxLogs=55 2024-11-22T08:55:23,624 INFO [RS:0;296fbbacac79:45617 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,45617,1732265721863/296fbbacac79%2C45617%2C1732265721863.1732265723608 2024-11-22T08:55:23,624 INFO [RS:1;296fbbacac79:34699 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,34699,1732265722113/296fbbacac79%2C34699%2C1732265722113.1732265723608 2024-11-22T08:55:23,625 DEBUG [RS:0;296fbbacac79:45617 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T08:55:23,625 DEBUG [RS:1;296fbbacac79:34699 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T08:55:23,775 DEBUG [296fbbacac79:40087 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-22T08:55:23,785 DEBUG [296fbbacac79:40087 {}] balancer.BalancerClusterState(204): Hosts are {296fbbacac79=0} racks are {/default-rack=0} 2024-11-22T08:55:23,788 DEBUG [296fbbacac79:40087 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T08:55:23,788 DEBUG [296fbbacac79:40087 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T08:55:23,788 DEBUG [296fbbacac79:40087 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T08:55:23,788 DEBUG [296fbbacac79:40087 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T08:55:23,788 INFO [296fbbacac79:40087 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T08:55:23,788 INFO [296fbbacac79:40087 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T08:55:23,788 DEBUG [296fbbacac79:40087 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T08:55:23,794 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=296fbbacac79,45617,1732265721863 2024-11-22T08:55:23,799 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 296fbbacac79,45617,1732265721863, state=OPENING 2024-11-22T08:55:23,804 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T08:55:23,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:23,806 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:23,806 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:23,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:23,808 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T08:55:23,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=296fbbacac79,45617,1732265721863}] 2024-11-22T08:55:23,978 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T08:55:23,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32919, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T08:55:23,990 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T08:55:23,991 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T08:55:23,991 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T08:55:23,993 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=296fbbacac79%2C45617%2C1732265721863.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,45617,1732265721863, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs, maxLogs=55 2024-11-22T08:55:24,011 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,45617,1732265721863/296fbbacac79%2C45617%2C1732265721863.meta.1732265723994.meta 2024-11-22T08:55:24,011 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T08:55:24,012 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T08:55:24,014 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T08:55:24,016 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T08:55:24,020 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T08:55:24,024 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T08:55:24,025 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:24,025 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T08:55:24,025 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T08:55:24,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T08:55:24,028 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T08:55:24,028 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:24,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:24,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T08:55:24,029 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T08:55:24,029 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:24,030 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:24,030 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T08:55:24,031 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T08:55:24,031 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:24,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:24,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T08:55:24,032 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T08:55:24,032 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:24,032 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:24,032 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T08:55:24,033 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740 2024-11-22T08:55:24,034 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740 2024-11-22T08:55:24,034 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T08:55:24,034 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T08:55:24,035 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T08:55:24,036 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T08:55:24,036 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67217664, jitterRate=0.001621246337890625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T08:55:24,036 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T08:55:24,038 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732265724026Writing region info on filesystem at 1732265724026Initializing all the Stores at 1732265724026Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265724026Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265724026Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265724027 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265724027Cleaning up temporary data from old regions at 1732265724034 (+7 ms)Running coprocessor post-open hooks at 1732265724037 (+3 ms)Region opened successfully at 1732265724038 (+1 ms) 2024-11-22T08:55:24,044 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732265723971 2024-11-22T08:55:24,055 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=296fbbacac79,45617,1732265721863 2024-11-22T08:55:24,056 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T08:55:24,056 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T08:55:24,057 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 296fbbacac79,45617,1732265721863, state=OPEN 2024-11-22T08:55:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:24,059 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:24,059 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:24,059 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:24,060 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=296fbbacac79,45617,1732265721863 2024-11-22T08:55:24,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T08:55:24,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=296fbbacac79,45617,1732265721863 in 251 msec 2024-11-22T08:55:24,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T08:55:24,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 706 msec 2024-11-22T08:55:24,069 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T08:55:24,069 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T08:55:24,087 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T08:55:24,089 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=296fbbacac79,45617,1732265721863, seqNum=-1] 2024-11-22T08:55:24,106 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T08:55:24,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60477, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T08:55:24,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 982 msec 2024-11-22T08:55:24,127 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732265724126, completionTime=-1 2024-11-22T08:55:24,129 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-22T08:55:24,130 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T08:55:24,151 INFO [master/296fbbacac79:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T08:55:24,151 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732265784151 2024-11-22T08:55:24,151 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732265844151 2024-11-22T08:55:24,152 INFO [master/296fbbacac79:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-11-22T08:55:24,154 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,40087,1732265720689-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:24,154 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,40087,1732265720689-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:24,154 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,40087,1732265720689-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:24,155 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-296fbbacac79:40087, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:24,156 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:24,156 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:24,162 DEBUG [master/296fbbacac79:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T08:55:24,185 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.909sec 2024-11-22T08:55:24,186 INFO [master/296fbbacac79:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T08:55:24,187 INFO [master/296fbbacac79:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T08:55:24,188 INFO [master/296fbbacac79:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T08:55:24,189 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T08:55:24,189 INFO [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T08:55:24,189 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,40087,1732265720689-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T08:55:24,190 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,40087,1732265720689-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T08:55:24,193 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T08:55:24,193 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T08:55:24,194 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,40087,1732265720689-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:24,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T08:55:24,225 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T08:55:24,226 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T08:55:24,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 296fbbacac79,40087,-1 for getting cluster id 2024-11-22T08:55:24,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T08:55:24,239 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48e01fbb-7ffd-4ac0-8c36-7c687543214c' 2024-11-22T08:55:24,241 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T08:55:24,242 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48e01fbb-7ffd-4ac0-8c36-7c687543214c" 2024-11-22T08:55:24,242 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e73f0ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T08:55:24,242 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [296fbbacac79,40087,-1] 2024-11-22T08:55:24,246 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T08:55:24,247 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:24,249 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T08:55:24,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T08:55:24,268 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T08:55:24,279 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=296fbbacac79,45617,1732265721863, seqNum=-1] 2024-11-22T08:55:24,280 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T08:55:24,282 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T08:55:24,300 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=296fbbacac79,40087,1732265720689 2024-11-22T08:55:24,303 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T08:55:24,308 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 296fbbacac79,40087,1732265720689 2024-11-22T08:55:24,310 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1029c010 2024-11-22T08:55:24,311 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T08:55:24,314 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T08:55:24,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732265724300', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T08:55:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732265724300 2024-11-22T08:55:24,327 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732265724300 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T08:55:24,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732265724300" procId is: 4 2024-11-22T08:55:24,330 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:24,331 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732265724300 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T08:55:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T08:55:24,344 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732265724300-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a5337380f2395a50c1120b5b872ae101, NAME => 'testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732265724300', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a 2024-11-22T08:55:24,350 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732265724300-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:24,350 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732265724300-pool-0 {}] regionserver.HRegion(1722): Closing a5337380f2395a50c1120b5b872ae101, disabling compactions & flushes 2024-11-22T08:55:24,350 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732265724300-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:24,350 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732265724300-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:24,350 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732265724300-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. after waiting 0 ms 2024-11-22T08:55:24,350 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732265724300-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:24,350 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732265724300-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:24,351 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732265724300-pool-0 {}] regionserver.HRegion(1676): Region close journal for a5337380f2395a50c1120b5b872ae101: Waiting for close lock at 1732265724350Disabling compacts and flushes for region at 1732265724350Disabling writes for close at 1732265724350Writing region close event to WAL at 1732265724350Closed at 1732265724350 2024-11-22T08:55:24,352 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732265724300 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T08:55:24,356 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732265724352"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732265724352"}]},"ts":"1732265724352"} 2024-11-22T08:55:24,360 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T08:55:24,362 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732265724300 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T08:55:24,364 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732265724300","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732265724362"}]},"ts":"1732265724362"} 2024-11-22T08:55:24,368 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732265724300, state=ENABLING in hbase:meta 2024-11-22T08:55:24,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {296fbbacac79=0} racks are {/default-rack=0} 2024-11-22T08:55:24,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T08:55:24,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T08:55:24,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T08:55:24,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T08:55:24,369 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T08:55:24,369 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T08:55:24,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T08:55:24,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, ASSIGN}] 2024-11-22T08:55:24,372 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, ASSIGN 2024-11-22T08:55:24,373 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, ASSIGN; state=OFFLINE, location=296fbbacac79,45617,1732265721863; forceNewPlan=false, retain=false 2024-11-22T08:55:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T08:55:24,526 INFO [296fbbacac79:40087 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T08:55:24,527 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a5337380f2395a50c1120b5b872ae101, regionState=OPENING, regionLocation=296fbbacac79,45617,1732265721863 2024-11-22T08:55:24,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, ASSIGN because future has completed 2024-11-22T08:55:24,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5337380f2395a50c1120b5b872ae101, server=296fbbacac79,45617,1732265721863}] 2024-11-22T08:55:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T08:55:24,688 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:24,688 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a5337380f2395a50c1120b5b872ae101, NAME => 'testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101.', STARTKEY => '', ENDKEY => ''} 2024-11-22T08:55:24,689 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732265724300 a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,689 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:24,689 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,690 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,691 INFO [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,697 INFO [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5337380f2395a50c1120b5b872ae101 columnFamilyName family 2024-11-22T08:55:24,697 DEBUG [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:24,698 INFO [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] regionserver.HStore(327): Store=a5337380f2395a50c1120b5b872ae101/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:24,698 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,700 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,700 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,700 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,700 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,702 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,708 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T08:55:24,708 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a5337380f2395a50c1120b5b872ae101; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69590923, jitterRate=0.03698556125164032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T08:55:24,709 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:24,710 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a5337380f2395a50c1120b5b872ae101: Running coprocessor pre-open hook at 1732265724690Writing region info on filesystem at 1732265724690Initializing all the Stores at 1732265724690Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265724691 (+1 ms)Cleaning up temporary data from old regions at 1732265724701 (+10 ms)Running coprocessor post-open hooks at 1732265724709 (+8 ms)Region opened successfully at 1732265724710 (+1 ms) 2024-11-22T08:55:24,711 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101., pid=6, masterSystemTime=1732265724684 2024-11-22T08:55:24,715 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:24,715 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:24,716 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a5337380f2395a50c1120b5b872ae101, regionState=OPEN, openSeqNum=2, regionLocation=296fbbacac79,45617,1732265721863 2024-11-22T08:55:24,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5337380f2395a50c1120b5b872ae101, server=296fbbacac79,45617,1732265721863 because future has completed 2024-11-22T08:55:24,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T08:55:24,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a5337380f2395a50c1120b5b872ae101, server=296fbbacac79,45617,1732265721863 in 190 msec 2024-11-22T08:55:24,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T08:55:24,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, ASSIGN in 355 msec 2024-11-22T08:55:24,728 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732265724300 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T08:55:24,729 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732265724300","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732265724728"}]},"ts":"1732265724728"} 2024-11-22T08:55:24,734 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732265724300, state=ENABLED in hbase:meta 2024-11-22T08:55:24,736 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732265724300 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T08:55:24,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732265724300 in 414 msec 2024-11-22T08:55:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T08:55:24,968 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732265724300 completed 2024-11-22T08:55:24,968 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732265724300 get assigned. Timeout = 60000ms 2024-11-22T08:55:24,972 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T08:55:24,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732265724300 assigned to meta. Checking AM states. 2024-11-22T08:55:24,983 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T08:55:24,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732265724300 assigned. 2024-11-22T08:55:24,993 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732265724300', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101., hostname=296fbbacac79,45617,1732265721863, seqNum=2] 2024-11-22T08:55:25,004 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a5337380f2395a50c1120b5b872ae101 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T08:55:25,027 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 97547772be7e4128972a6fccbed0ded3 2024-11-22T08:55:25,062 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/.tmp/family/97547772be7e4128972a6fccbed0ded3 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T08:55:25,067 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/.tmp/family/97547772be7e4128972a6fccbed0ded3 2024-11-22T08:55:25,100 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/.tmp/family/97547772be7e4128972a6fccbed0ded3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3 2024-11-22T08:55:25,101 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3, delay=988 ms 2024-11-22T08:55:25,104 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T08:55:25,110 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for a5337380f2395a50c1120b5b872ae101 in 106ms, sequenceid=6, compaction requested=false 2024-11-22T08:55:25,111 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732265724300' 2024-11-22T08:55:25,113 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a5337380f2395a50c1120b5b872ae101: 2024-11-22T08:55:26,095 DEBUG [hfile-prefetch-1732265725101 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 97547772be7e4128972a6fccbed0ded3_0. 2024-11-22T08:55:26,100 DEBUG [hfile-prefetch-1732265725101 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 97547772be7e4128972a6fccbed0ded3_117. 2024-11-22T08:55:26,101 DEBUG [hfile-prefetch-1732265725101 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3, with total blocks 2, and data blocks 1 2024-11-22T08:55:26,101 DEBUG [hfile-prefetch-1732265725101 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T08:55:26,101 DEBUG [hfile-prefetch-1732265725101 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T08:55:26,102 DEBUG [hfile-prefetch-1732265725101 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3. Will sleep 100ms and try the verification again. 2024-11-22T08:55:26,108 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T08:55:26,108 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T08:55:26,109 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T08:55:26,109 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T08:55:26,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=a5337380f2395a50c1120b5b872ae101, source=296fbbacac79,45617,1732265721863, destination=296fbbacac79,34699,1732265722113, warming up region on 296fbbacac79,34699,1732265722113 2024-11-22T08:55:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T08:55:26,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=a5337380f2395a50c1120b5b872ae101, source=296fbbacac79,45617,1732265721863, destination=296fbbacac79,34699,1732265722113, running balancer 2024-11-22T08:55:26,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39937, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T08:55:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, REOPEN/MOVE 2024-11-22T08:55:26,131 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, REOPEN/MOVE 2024-11-22T08:55:26,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34699 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,132 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a5337380f2395a50c1120b5b872ae101, regionState=CLOSING, regionLocation=296fbbacac79,45617,1732265721863 2024-11-22T08:55:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34699 {}] regionserver.HRegion(7855): Warmup {ENCODED => a5337380f2395a50c1120b5b872ae101, NAME => 'testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101.', STARTKEY => '', ENDKEY => ''} 2024-11-22T08:55:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34699 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:26,133 INFO [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,134 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T08:55:26,134 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732265726108 2024-11-22T08:55:26,134 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T08:55:26,134 INFO [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5337380f2395a50c1120b5b872ae101 columnFamilyName family 2024-11-22T08:55:26,135 DEBUG [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:26,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, REOPEN/MOVE because future has completed 2024-11-22T08:55:26,136 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-22T08:55:26,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5337380f2395a50c1120b5b872ae101, server=296fbbacac79,45617,1732265721863}] 2024-11-22T08:55:26,138 DEBUG [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3 2024-11-22T08:55:26,138 INFO [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] regionserver.HStore(327): Store=a5337380f2395a50c1120b5b872ae101/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34699 {}] regionserver.HRegion(1722): Closing a5337380f2395a50c1120b5b872ae101, disabling compactions & flushes 2024-11-22T08:55:26,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34699 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34699 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34699 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. after waiting 0 ms 2024-11-22T08:55:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34699 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,140 DEBUG [StoreFileCloser-a5337380f2395a50c1120b5b872ae101-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3 2024-11-22T08:55:26,140 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34699 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34699 {}] regionserver.HRegion(1676): Region close journal for a5337380f2395a50c1120b5b872ae101: Waiting for close lock at 1732265726138Disabling compacts and flushes for region at 1732265726138Disabling writes for close at 1732265726138Writing region close event to WAL at 1732265726140 (+2 ms)Closed at 1732265726140 2024-11-22T08:55:26,142 WARN [hfile-prefetch-1732265725101 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-22T08:55:26,143 DEBUG [hfile-prefetch-1732265725101 {}] hfile.PrefetchExecutor(121): Prefetch completed for 97547772be7e4128972a6fccbed0ded3 2024-11-22T08:55:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-22T08:55:26,296 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,297 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-22T08:55:26,298 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing a5337380f2395a50c1120b5b872ae101, disabling compactions & flushes 2024-11-22T08:55:26,298 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,298 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,299 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. after waiting 0 ms 2024-11-22T08:55:26,299 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,302 DEBUG [StoreFileCloser-a5337380f2395a50c1120b5b872ae101-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 97547772be7e4128972a6fccbed0ded3, starting offset: 0, end offset: 9223372036854775807 2024-11-22T08:55:26,305 DEBUG [StoreFileCloser-a5337380f2395a50c1120b5b872ae101-family-1 {}] bucket.BucketCache(856): removed key 97547772be7e4128972a6fccbed0ded3_0 from back map with offset lock 2094080 in the evict process 2024-11-22T08:55:26,310 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T08:55:26,310 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,311 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for a5337380f2395a50c1120b5b872ae101: Waiting for close lock at 1732265726298Running coprocessor pre-close hooks at 1732265726298Disabling compacts and flushes for region at 1732265726298Disabling writes for close at 1732265726299 (+1 ms)Writing region close event to WAL at 1732265726305 (+6 ms)Running coprocessor post-close hooks at 1732265726310 (+5 ms)Closed at 1732265726310 2024-11-22T08:55:26,311 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding a5337380f2395a50c1120b5b872ae101 move to 296fbbacac79,34699,1732265722113 record at close sequenceid=6 2024-11-22T08:55:26,314 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,315 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a5337380f2395a50c1120b5b872ae101, regionState=CLOSED 2024-11-22T08:55:26,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5337380f2395a50c1120b5b872ae101, server=296fbbacac79,45617,1732265721863 because future has completed 2024-11-22T08:55:26,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T08:55:26,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure a5337380f2395a50c1120b5b872ae101, server=296fbbacac79,45617,1732265721863 in 183 msec 2024-11-22T08:55:26,321 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, REOPEN/MOVE; state=CLOSED, location=296fbbacac79,34699,1732265722113; forceNewPlan=false, retain=false 2024-11-22T08:55:26,334 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T08:55:26,334 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T08:55:26,335 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T08:55:26,335 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T08:55:26,338 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T08:55:26,338 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732265726335 2024-11-22T08:55:26,338 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T08:55:26,472 INFO [296fbbacac79:40087 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T08:55:26,472 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a5337380f2395a50c1120b5b872ae101, regionState=OPENING, regionLocation=296fbbacac79,34699,1732265722113 2024-11-22T08:55:26,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, REOPEN/MOVE because future has completed 2024-11-22T08:55:26,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5337380f2395a50c1120b5b872ae101, server=296fbbacac79,34699,1732265722113}] 2024-11-22T08:55:26,641 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,642 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => a5337380f2395a50c1120b5b872ae101, NAME => 'testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101.', STARTKEY => '', ENDKEY => ''} 2024-11-22T08:55:26,643 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732265724300 a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,643 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:26,644 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,644 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,645 INFO [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,646 INFO [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5337380f2395a50c1120b5b872ae101 columnFamilyName family 2024-11-22T08:55:26,646 DEBUG [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:26,648 DEBUG [StoreFileOpener-a5337380f2395a50c1120b5b872ae101-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3, delay=973 ms 2024-11-22T08:55:26,649 DEBUG [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3 2024-11-22T08:55:26,650 INFO [StoreOpener-a5337380f2395a50c1120b5b872ae101-1 {}] regionserver.HStore(327): Store=a5337380f2395a50c1120b5b872ae101/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:26,650 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,651 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,651 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,651 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,651 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,655 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,656 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened a5337380f2395a50c1120b5b872ae101; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65275642, jitterRate=-0.027317136526107788}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T08:55:26,656 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,657 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for a5337380f2395a50c1120b5b872ae101: Running coprocessor pre-open hook at 1732265726644Writing region info on filesystem at 1732265726644Initializing all the Stores at 1732265726645 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265726645Cleaning up temporary data from old regions at 1732265726652 (+7 ms)Running coprocessor post-open hooks at 1732265726656 (+4 ms)Region opened successfully at 1732265726657 (+1 ms) 2024-11-22T08:55:26,657 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101., pid=9, masterSystemTime=1732265726635 2024-11-22T08:55:26,660 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,660 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,661 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a5337380f2395a50c1120b5b872ae101, regionState=OPEN, openSeqNum=10, regionLocation=296fbbacac79,34699,1732265722113 2024-11-22T08:55:26,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5337380f2395a50c1120b5b872ae101, server=296fbbacac79,34699,1732265722113 because future has completed 2024-11-22T08:55:26,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-22T08:55:26,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure a5337380f2395a50c1120b5b872ae101, server=296fbbacac79,34699,1732265722113 in 185 msec 2024-11-22T08:55:26,668 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732265724300, region=a5337380f2395a50c1120b5b872ae101, REOPEN/MOVE in 538 msec 2024-11-22T08:55:26,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T08:55:26,684 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T08:55:26,684 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T08:55:26,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:26,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:26,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T08:55:26,689 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T08:55:26,689 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2143720667, stopped=false 2024-11-22T08:55:26,690 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=296fbbacac79,40087,1732265720689 2024-11-22T08:55:26,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:26,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:26,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:26,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:26,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:26,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:26,692 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T08:55:26,692 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:26,692 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:26,692 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:26,693 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T08:55:26,693 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T08:55:26,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:26,694 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '296fbbacac79,45617,1732265721863' ***** 2024-11-22T08:55:26,694 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T08:55:26,694 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '296fbbacac79,34699,1732265722113' ***** 2024-11-22T08:55:26,694 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T08:55:26,694 INFO [RS:0;296fbbacac79:45617 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T08:55:26,694 INFO [RS:1;296fbbacac79:34699 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T08:55:26,694 INFO [RS:0;296fbbacac79:45617 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/bucket.persistence 2024-11-22T08:55:26,694 INFO [RS:1;296fbbacac79:34699 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/bucket.persistence 2024-11-22T08:55:26,695 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T08:55:26,695 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T08:55:26,695 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T08:55:26,695 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T08:55:26,695 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T08:55:26,695 INFO [RS:0;296fbbacac79:45617 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T08:55:26,695 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T08:55:26,695 INFO [RS:1;296fbbacac79:34699 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T08:55:26,695 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T08:55:26,695 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T08:55:26,705 DEBUG [RS:0;296fbbacac79:45617 {}] bucket.BucketCache(1487): Thread RS:0;296fbbacac79:45617 started persisting bucket cache to file 2024-11-22T08:55:26,705 DEBUG [RS:1;296fbbacac79:34699 {}] bucket.BucketCache(1487): Thread RS:1;296fbbacac79:34699 started persisting bucket cache to file 2024-11-22T08:55:26,705 DEBUG [RS:0;296fbbacac79:45617 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T08:55:26,705 DEBUG [RS:1;296fbbacac79:34699 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T08:55:26,705 DEBUG [RS:0;296fbbacac79:45617 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T08:55:26,705 DEBUG [RS:1;296fbbacac79:34699 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T08:55:26,708 DEBUG [RS:1;296fbbacac79:34699 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T08:55:26,708 DEBUG [RS:1;296fbbacac79:34699 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732265726705 2024-11-22T08:55:26,708 DEBUG [RS:0;296fbbacac79:45617 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T08:55:26,708 DEBUG [RS:1;296fbbacac79:34699 {}] bucket.BucketCache(1509): Thread RS:1;296fbbacac79:34699 finished persisting bucket cache to file, renaming 2024-11-22T08:55:26,708 DEBUG [RS:0;296fbbacac79:45617 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732265726705 2024-11-22T08:55:26,708 DEBUG [RS:0;296fbbacac79:45617 {}] bucket.BucketCache(1509): Thread RS:0;296fbbacac79:45617 finished persisting bucket cache to file, renaming 2024-11-22T08:55:26,708 WARN [RS:0;296fbbacac79:45617 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-22T08:55:26,708 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T08:55:26,709 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T08:55:26,709 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T08:55:26,709 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T08:55:26,709 INFO [RS:1;296fbbacac79:34699 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T08:55:26,709 INFO [RS:0;296fbbacac79:45617 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T08:55:26,709 INFO [RS:1;296fbbacac79:34699 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T08:55:26,709 INFO [RS:0;296fbbacac79:45617 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T08:55:26,709 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(959): stopping server 296fbbacac79,45617,1732265721863 2024-11-22T08:55:26,709 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(3091): Received CLOSE for a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,709 INFO [RS:0;296fbbacac79:45617 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T08:55:26,709 INFO [RS:0;296fbbacac79:45617 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;296fbbacac79:45617. 2024-11-22T08:55:26,710 DEBUG [RS:0;296fbbacac79:45617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T08:55:26,710 DEBUG [RS:0;296fbbacac79:45617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:26,710 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(959): stopping server 296fbbacac79,34699,1732265722113 2024-11-22T08:55:26,710 INFO [RS:1;296fbbacac79:34699 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T08:55:26,710 INFO [RS:0;296fbbacac79:45617 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T08:55:26,710 INFO [RS:0;296fbbacac79:45617 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T08:55:26,710 INFO [RS:1;296fbbacac79:34699 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;296fbbacac79:34699. 2024-11-22T08:55:26,710 INFO [RS:0;296fbbacac79:45617 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T08:55:26,710 DEBUG [RS:1;296fbbacac79:34699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T08:55:26,710 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T08:55:26,710 DEBUG [RS:1;296fbbacac79:34699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:26,710 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T08:55:26,710 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T08:55:26,710 DEBUG [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(1325): Online Regions={a5337380f2395a50c1120b5b872ae101=testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101.} 2024-11-22T08:55:26,711 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a5337380f2395a50c1120b5b872ae101, disabling compactions & flushes 2024-11-22T08:55:26,711 DEBUG [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(1351): Waiting on a5337380f2395a50c1120b5b872ae101 2024-11-22T08:55:26,711 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,711 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T08:55:26,711 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,711 DEBUG [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T08:55:26,711 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. after waiting 0 ms 2024-11-22T08:55:26,711 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,711 DEBUG [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T08:55:26,711 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T08:55:26,711 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T08:55:26,711 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T08:55:26,711 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T08:55:26,711 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T08:55:26,711 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T08:55:26,711 DEBUG [StoreFileCloser-a5337380f2395a50c1120b5b872ae101-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/family/97547772be7e4128972a6fccbed0ded3 2024-11-22T08:55:26,712 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-22T08:55:26,716 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnRegionMove1732265724300/a5337380f2395a50c1120b5b872ae101/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-22T08:55:26,716 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,716 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a5337380f2395a50c1120b5b872ae101: Waiting for close lock at 1732265726710Running coprocessor pre-close hooks at 1732265726711 (+1 ms)Disabling compacts and flushes for region at 1732265726711Disabling writes for close at 1732265726711Writing region close event to WAL at 1732265726712 (+1 ms)Running coprocessor post-close hooks at 1732265726716 (+4 ms)Closed at 1732265726716 2024-11-22T08:55:26,716 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101. 2024-11-22T08:55:26,721 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 187384988b59484d94254487345a959b 2024-11-22T08:55:26,737 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/info/187384988b59484d94254487345a959b is 205, key is testBlockEvictionOnRegionMove1732265724300,,1732265724315.a5337380f2395a50c1120b5b872ae101./info:regioninfo/1732265726660/Put/seqid=0 2024-11-22T08:55:26,738 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/info/187384988b59484d94254487345a959b 2024-11-22T08:55:26,746 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e254060e5d1846c5ad1801ee02b35104 2024-11-22T08:55:26,761 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/ns/e254060e5d1846c5ad1801ee02b35104 is 43, key is default/ns:d/1732265724112/Put/seqid=0 2024-11-22T08:55:26,762 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/ns/e254060e5d1846c5ad1801ee02b35104 2024-11-22T08:55:26,764 INFO [regionserver/296fbbacac79:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T08:55:26,764 INFO [regionserver/296fbbacac79:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T08:55:26,771 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3ba48035e7c54e08b06f2556e2d9100b 2024-11-22T08:55:26,787 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/table/3ba48035e7c54e08b06f2556e2d9100b is 78, key is testBlockEvictionOnRegionMove1732265724300/table:state/1732265724728/Put/seqid=0 2024-11-22T08:55:26,788 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/table/3ba48035e7c54e08b06f2556e2d9100b 2024-11-22T08:55:26,791 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/info/187384988b59484d94254487345a959b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/187384988b59484d94254487345a959b 2024-11-22T08:55:26,793 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/187384988b59484d94254487345a959b, delay=934 ms 2024-11-22T08:55:26,795 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/187384988b59484d94254487345a959b, entries=15, sequenceid=15, filesize=7.6 K 2024-11-22T08:55:26,796 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/ns/e254060e5d1846c5ad1801ee02b35104 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/e254060e5d1846c5ad1801ee02b35104 2024-11-22T08:55:26,797 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/e254060e5d1846c5ad1801ee02b35104, delay=973 ms 2024-11-22T08:55:26,799 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/e254060e5d1846c5ad1801ee02b35104, entries=2, sequenceid=15, filesize=5.0 K 2024-11-22T08:55:26,799 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/table/3ba48035e7c54e08b06f2556e2d9100b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/3ba48035e7c54e08b06f2556e2d9100b 2024-11-22T08:55:26,801 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/3ba48035e7c54e08b06f2556e2d9100b, delay=920 ms 2024-11-22T08:55:26,802 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/3ba48035e7c54e08b06f2556e2d9100b, entries=2, sequenceid=15, filesize=5.3 K 2024-11-22T08:55:26,803 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false 2024-11-22T08:55:26,803 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T08:55:26,805 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/187384988b59484d94254487345a959b 2024-11-22T08:55:26,806 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/e254060e5d1846c5ad1801ee02b35104 2024-11-22T08:55:26,807 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/3ba48035e7c54e08b06f2556e2d9100b 2024-11-22T08:55:26,811 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-22T08:55:26,812 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T08:55:26,813 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T08:55:26,813 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732265726711Running coprocessor pre-close hooks at 1732265726711Disabling compacts and flushes for region at 1732265726711Disabling writes for close at 1732265726711Obtaining lock to block concurrent updates at 1732265726712 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732265726712Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732265726712Flushing stores of hbase:meta,,1.1588230740 at 1732265726712Flushing 1588230740/info: creating writer at 1732265726712Flushing 1588230740/info: appending metadata at 1732265726736 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732265726736Flushing 1588230740/ns: creating writer at 1732265726741 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732265726759 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732265726759Flushing 1588230740/table: creating writer at 1732265726766 (+7 ms)Flushing 1588230740/table: appending metadata at 1732265726786 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732265726786Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14ccf897: reopening flushed file at 1732265726791 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33b1ece6: reopening flushed file at 1732265726795 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a460c8: reopening flushed file at 1732265726799 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false at 1732265726803 (+4 ms)Writing region close event to WAL at 1732265726807 (+4 ms)Running coprocessor post-close hooks at 1732265726812 (+5 ms)Closed at 1732265726813 (+1 ms) 2024-11-22T08:55:26,813 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T08:55:26,911 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(976): stopping server 296fbbacac79,34699,1732265722113; all regions closed. 2024-11-22T08:55:26,911 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(976): stopping server 296fbbacac79,45617,1732265721863; all regions closed. 2024-11-22T08:55:26,916 DEBUG [RS:1;296fbbacac79:34699 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs 2024-11-22T08:55:26,916 DEBUG [RS:0;296fbbacac79:45617 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs 2024-11-22T08:55:26,916 INFO [RS:1;296fbbacac79:34699 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 296fbbacac79%2C34699%2C1732265722113:(num 1732265723608) 2024-11-22T08:55:26,916 INFO [RS:0;296fbbacac79:45617 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 296fbbacac79%2C45617%2C1732265721863.meta:.meta(num 1732265723994) 2024-11-22T08:55:26,916 DEBUG [RS:1;296fbbacac79:34699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:26,916 INFO [RS:1;296fbbacac79:34699 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T08:55:26,916 INFO [RS:1;296fbbacac79:34699 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T08:55:26,916 INFO [RS:1;296fbbacac79:34699 {}] hbase.ChoreService(370): Chore service for: regionserver/296fbbacac79:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T08:55:26,917 INFO [RS:1;296fbbacac79:34699 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T08:55:26,917 INFO [RS:1;296fbbacac79:34699 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T08:55:26,917 INFO [regionserver/296fbbacac79:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T08:55:26,917 INFO [RS:1;296fbbacac79:34699 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T08:55:26,917 INFO [RS:1;296fbbacac79:34699 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T08:55:26,918 INFO [RS:1;296fbbacac79:34699 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34699 2024-11-22T08:55:26,918 DEBUG [RS:0;296fbbacac79:45617 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs 2024-11-22T08:55:26,918 INFO [RS:0;296fbbacac79:45617 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 296fbbacac79%2C45617%2C1732265721863:(num 1732265723608) 2024-11-22T08:55:26,918 DEBUG [RS:0;296fbbacac79:45617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:26,918 INFO [RS:0;296fbbacac79:45617 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T08:55:26,919 INFO [RS:0;296fbbacac79:45617 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T08:55:26,919 INFO [RS:0;296fbbacac79:45617 {}] hbase.ChoreService(370): Chore service for: regionserver/296fbbacac79:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T08:55:26,919 INFO [RS:0;296fbbacac79:45617 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T08:55:26,919 INFO [regionserver/296fbbacac79:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T08:55:26,920 INFO [RS:0;296fbbacac79:45617 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45617 2024-11-22T08:55:26,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T08:55:26,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/296fbbacac79,45617,1732265721863 2024-11-22T08:55:26,923 INFO [RS:0;296fbbacac79:45617 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T08:55:26,923 INFO [RS:1;296fbbacac79:34699 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T08:55:26,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/296fbbacac79,34699,1732265722113 2024-11-22T08:55:26,925 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [296fbbacac79,34699,1732265722113] 2024-11-22T08:55:26,926 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/296fbbacac79,34699,1732265722113 already deleted, retry=false 2024-11-22T08:55:26,927 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 296fbbacac79,34699,1732265722113 expired; onlineServers=1 2024-11-22T08:55:26,927 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [296fbbacac79,45617,1732265721863] 2024-11-22T08:55:26,928 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/296fbbacac79,45617,1732265721863 already deleted, retry=false 2024-11-22T08:55:26,928 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 296fbbacac79,45617,1732265721863 expired; onlineServers=0 2024-11-22T08:55:26,928 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '296fbbacac79,40087,1732265720689' ***** 2024-11-22T08:55:26,928 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T08:55:26,929 INFO [M:0;296fbbacac79:40087 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T08:55:26,929 INFO [M:0;296fbbacac79:40087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T08:55:26,929 DEBUG [M:0;296fbbacac79:40087 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T08:55:26,929 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T08:55:26,929 DEBUG [M:0;296fbbacac79:40087 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T08:55:26,929 DEBUG [master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.small.0-1732265723267 {}] cleaner.HFileCleaner(306): Exit Thread[master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.small.0-1732265723267,5,FailOnTimeoutGroup] 2024-11-22T08:55:26,929 DEBUG [master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.large.0-1732265723266 {}] cleaner.HFileCleaner(306): Exit Thread[master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.large.0-1732265723266,5,FailOnTimeoutGroup] 2024-11-22T08:55:26,929 INFO [M:0;296fbbacac79:40087 {}] hbase.ChoreService(370): Chore service for: master/296fbbacac79:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T08:55:26,929 INFO [M:0;296fbbacac79:40087 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T08:55:26,929 DEBUG [M:0;296fbbacac79:40087 {}] master.HMaster(1795): Stopping service threads 2024-11-22T08:55:26,930 INFO [M:0;296fbbacac79:40087 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T08:55:26,930 INFO [M:0;296fbbacac79:40087 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T08:55:26,930 INFO [M:0;296fbbacac79:40087 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T08:55:26,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T08:55:26,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:26,930 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T08:55:26,931 DEBUG [M:0;296fbbacac79:40087 {}] zookeeper.ZKUtil(347): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T08:55:26,931 WARN [M:0;296fbbacac79:40087 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T08:55:26,931 INFO [M:0;296fbbacac79:40087 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/.lastflushedseqids 2024-11-22T08:55:26,939 INFO [M:0;296fbbacac79:40087 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T08:55:26,939 INFO [M:0;296fbbacac79:40087 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T08:55:26,940 DEBUG [M:0;296fbbacac79:40087 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T08:55:26,940 INFO [M:0;296fbbacac79:40087 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:26,940 DEBUG [M:0;296fbbacac79:40087 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:26,940 DEBUG [M:0;296fbbacac79:40087 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T08:55:26,940 DEBUG [M:0;296fbbacac79:40087 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:26,940 INFO [M:0;296fbbacac79:40087 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-22T08:55:26,949 DEBUG [M:0;296fbbacac79:40087 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6e965414345443cfa9fbaeea1b44193b 2024-11-22T08:55:26,963 DEBUG [M:0;296fbbacac79:40087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6e965414345443cfa9fbaeea1b44193b is 82, key is hbase:meta,,1/info:regioninfo/1732265724054/Put/seqid=0 2024-11-22T08:55:26,963 INFO [M:0;296fbbacac79:40087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6e965414345443cfa9fbaeea1b44193b 2024-11-22T08:55:26,971 DEBUG [M:0;296fbbacac79:40087 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 73d67a62807f4a9993f1648bb7a3af79 2024-11-22T08:55:26,984 DEBUG [M:0;296fbbacac79:40087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/73d67a62807f4a9993f1648bb7a3af79 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732265724737/Put/seqid=0 2024-11-22T08:55:26,985 INFO [M:0;296fbbacac79:40087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/73d67a62807f4a9993f1648bb7a3af79 2024-11-22T08:55:26,993 DEBUG [M:0;296fbbacac79:40087 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 81948a8a9d17487d9b4f509901e009e8 2024-11-22T08:55:27,007 DEBUG [M:0;296fbbacac79:40087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/81948a8a9d17487d9b4f509901e009e8 is 69, key is 296fbbacac79,34699,1732265722113/rs:state/1732265723363/Put/seqid=0 2024-11-22T08:55:27,007 INFO [M:0;296fbbacac79:40087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/81948a8a9d17487d9b4f509901e009e8 2024-11-22T08:55:27,009 DEBUG [M:0;296fbbacac79:40087 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6e965414345443cfa9fbaeea1b44193b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6e965414345443cfa9fbaeea1b44193b 2024-11-22T08:55:27,011 INFO [M:0;296fbbacac79:40087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6e965414345443cfa9fbaeea1b44193b, entries=8, sequenceid=84, filesize=5.5 K 2024-11-22T08:55:27,012 DEBUG [M:0;296fbbacac79:40087 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/73d67a62807f4a9993f1648bb7a3af79 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/73d67a62807f4a9993f1648bb7a3af79 2024-11-22T08:55:27,014 INFO [M:0;296fbbacac79:40087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/73d67a62807f4a9993f1648bb7a3af79, entries=9, sequenceid=84, filesize=6.5 K 2024-11-22T08:55:27,015 DEBUG [M:0;296fbbacac79:40087 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/81948a8a9d17487d9b4f509901e009e8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/81948a8a9d17487d9b4f509901e009e8 2024-11-22T08:55:27,017 INFO [M:0;296fbbacac79:40087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/81948a8a9d17487d9b4f509901e009e8, entries=2, sequenceid=84, filesize=5.1 K 2024-11-22T08:55:27,018 INFO [M:0;296fbbacac79:40087 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=84, compaction requested=false 2024-11-22T08:55:27,020 INFO [M:0;296fbbacac79:40087 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:27,020 DEBUG [M:0;296fbbacac79:40087 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732265726940Disabling compacts and flushes for region at 1732265726940Disabling writes for close at 1732265726940Obtaining lock to block concurrent updates at 1732265726940Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732265726940Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732265726941 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732265726941Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732265726941Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732265726962 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732265726962Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732265726965 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732265726984 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732265726984Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732265726987 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732265727006 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732265727006Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ffd2fed: reopening flushed file at 1732265727009 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e0cbafd: reopening flushed file at 1732265727011 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65d35a8: reopening flushed file at 1732265727014 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=84, compaction requested=false at 1732265727018 (+4 ms)Writing region close event to WAL at 1732265727020 (+2 ms)Closed at 1732265727020 2024-11-22T08:55:27,022 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T08:55:27,022 INFO [M:0;296fbbacac79:40087 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T08:55:27,022 INFO [M:0;296fbbacac79:40087 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40087 2024-11-22T08:55:27,022 INFO [M:0;296fbbacac79:40087 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T08:55:27,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:27,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:27,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34699-0x10066a495d70002, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:27,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45617-0x10066a495d70001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:27,026 INFO [RS:0;296fbbacac79:45617 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T08:55:27,026 INFO [RS:1;296fbbacac79:34699 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T08:55:27,026 INFO [RS:0;296fbbacac79:45617 {}] regionserver.HRegionServer(1031): Exiting; stopping=296fbbacac79,45617,1732265721863; zookeeper connection closed. 2024-11-22T08:55:27,026 INFO [RS:1;296fbbacac79:34699 {}] regionserver.HRegionServer(1031): Exiting; stopping=296fbbacac79,34699,1732265722113; zookeeper connection closed. 2024-11-22T08:55:27,026 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1439a851 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1439a851 2024-11-22T08:55:27,026 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1435676a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1435676a 2024-11-22T08:55:27,027 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-22T08:55:27,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:27,124 INFO [M:0;296fbbacac79:40087 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T08:55:27,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x10066a495d70000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:27,130 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T08:55:27,149 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T08:55:27,149 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:27,156 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/296fbbacac79:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732265726797 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732265726648 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/296fbbacac79:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/296fbbacac79:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732265726793 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732265725101 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/296fbbacac79:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=288 (was 313), ProcessCount=11 (was 11), AvailableMemoryMB=5174 (was 5871) 2024-11-22T08:55:27,162 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=288, ProcessCount=11, AvailableMemoryMB=5174 2024-11-22T08:55:27,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/hadoop-log-dir so I do NOT create it in target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192 2024-11-22T08:55:27,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/hadoop-tmp-dir so I do NOT create it in target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192 2024-11-22T08:55:27,162 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:27,165 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/cluster_dd9023c3-43e2-277b-26f1-f2e6f3aac3d7, deleteOnExit=true 2024-11-22T08:55:27,168 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/cluster_dd9023c3-43e2-277b-26f1-f2e6f3aac3d7/zookeeper_0, clientPort=53512, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/cluster_dd9023c3-43e2-277b-26f1-f2e6f3aac3d7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/cluster_dd9023c3-43e2-277b-26f1-f2e6f3aac3d7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T08:55:27,169 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53512 2024-11-22T08:55:27,179 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a with version=8 2024-11-22T08:55:27,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/hbase-staging 2024-11-22T08:55:27,182 INFO [Time-limited test {}] client.ConnectionUtils(128): master/296fbbacac79:0 server-side Connection retries=45 2024-11-22T08:55:27,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:27,183 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:27,183 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T08:55:27,183 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:27,183 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T08:55:27,183 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T08:55:27,183 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T08:55:27,184 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39331 2024-11-22T08:55:27,186 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39331 connecting to ZooKeeper ensemble=127.0.0.1:53512 2024-11-22T08:55:27,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:393310x0, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T08:55:27,191 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39331-0x10066a4b12e0000 connected 2024-11-22T08:55:27,208 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:27,208 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:27,210 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:27,210 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a, hbase.cluster.distributed=false 2024-11-22T08:55:27,213 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T08:55:27,214 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39331 2024-11-22T08:55:27,214 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39331 2024-11-22T08:55:27,215 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39331 2024-11-22T08:55:27,215 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39331 2024-11-22T08:55:27,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39331 2024-11-22T08:55:27,234 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/296fbbacac79:0 server-side Connection retries=45 2024-11-22T08:55:27,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:27,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:27,235 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T08:55:27,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:27,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T08:55:27,235 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T08:55:27,235 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T08:55:27,236 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34493 2024-11-22T08:55:27,237 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34493 connecting to ZooKeeper ensemble=127.0.0.1:53512 2024-11-22T08:55:27,237 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:27,238 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:27,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344930x0, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T08:55:27,240 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34493-0x10066a4b12e0001 connected 2024-11-22T08:55:27,240 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:27,241 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T08:55:27,241 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.cache 2024-11-22T08:55:27,241 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T08:55:27,242 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T08:55:27,242 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T08:55:27,242 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T08:55:27,242 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T08:55:27,242 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T08:55:27,243 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T08:55:27,243 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.persistence is complete. 2024-11-22T08:55:27,243 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T08:55:27,244 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T08:55:27,245 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T08:55:27,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34493 2024-11-22T08:55:27,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34493 2024-11-22T08:55:27,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34493 2024-11-22T08:55:27,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34493 2024-11-22T08:55:27,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34493 2024-11-22T08:55:27,268 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/296fbbacac79:0 server-side Connection retries=45 2024-11-22T08:55:27,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:27,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:27,269 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T08:55:27,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:27,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T08:55:27,269 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T08:55:27,269 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T08:55:27,270 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41191 2024-11-22T08:55:27,271 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41191 connecting to ZooKeeper ensemble=127.0.0.1:53512 2024-11-22T08:55:27,272 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:27,272 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:27,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:411910x0, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T08:55:27,275 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41191-0x10066a4b12e0002 connected 2024-11-22T08:55:27,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:27,275 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T08:55:27,277 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.cache 2024-11-22T08:55:27,277 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T08:55:27,277 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T08:55:27,277 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T08:55:27,278 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T08:55:27,278 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T08:55:27,278 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T08:55:27,278 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T08:55:27,278 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T08:55:27,278 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.persistence is complete. 2024-11-22T08:55:27,279 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T08:55:27,280 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T08:55:27,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41191 2024-11-22T08:55:27,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41191 2024-11-22T08:55:27,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41191 2024-11-22T08:55:27,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41191 2024-11-22T08:55:27,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41191 2024-11-22T08:55:27,298 DEBUG [M:0;296fbbacac79:39331 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;296fbbacac79:39331 2024-11-22T08:55:27,298 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/296fbbacac79,39331,1732265727182 2024-11-22T08:55:27,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:27,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:27,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:27,300 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/296fbbacac79,39331,1732265727182 2024-11-22T08:55:27,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T08:55:27,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T08:55:27,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,303 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T08:55:27,304 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/296fbbacac79,39331,1732265727182 from backup master directory 2024-11-22T08:55:27,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/296fbbacac79,39331,1732265727182 2024-11-22T08:55:27,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:27,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:27,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T08:55:27,306 WARN [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T08:55:27,306 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=296fbbacac79,39331,1732265727182 2024-11-22T08:55:27,308 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/hbase.id] with ID: 05f9bea9-a0cc-4981-ae4a-0b352f94b7d9 2024-11-22T08:55:27,308 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/.tmp/hbase.id 2024-11-22T08:55:27,313 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/hbase.id] 2024-11-22T08:55:27,344 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:27,344 INFO [master/296fbbacac79:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T08:55:27,347 INFO [master/296fbbacac79:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-22T08:55:27,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,353 INFO [master/296fbbacac79:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T08:55:27,354 INFO [master/296fbbacac79:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T08:55:27,354 INFO [master/296fbbacac79:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T08:55:27,372 INFO [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store 2024-11-22T08:55:27,377 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:27,377 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T08:55:27,377 INFO [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:27,377 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:27,377 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T08:55:27,377 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:27,377 INFO [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:27,377 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732265727377Disabling compacts and flushes for region at 1732265727377Disabling writes for close at 1732265727377Writing region close event to WAL at 1732265727377Closed at 1732265727377 2024-11-22T08:55:27,378 WARN [master/296fbbacac79:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/.initializing 2024-11-22T08:55:27,378 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/WALs/296fbbacac79,39331,1732265727182 2024-11-22T08:55:27,382 INFO [master/296fbbacac79:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T08:55:27,385 INFO [master/296fbbacac79:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=296fbbacac79%2C39331%2C1732265727182, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/WALs/296fbbacac79,39331,1732265727182, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/oldWALs, maxLogs=10 2024-11-22T08:55:27,401 INFO [master/296fbbacac79:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/WALs/296fbbacac79,39331,1732265727182/296fbbacac79%2C39331%2C1732265727182.1732265727385 2024-11-22T08:55:27,402 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T08:55:27,402 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T08:55:27,402 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:27,402 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,402 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,403 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T08:55:27,407 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:27,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,411 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T08:55:27,411 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,411 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:27,411 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,414 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T08:55:27,414 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:27,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,420 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T08:55:27,420 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,420 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:27,420 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,421 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,421 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,422 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,422 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,422 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T08:55:27,422 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T08:55:27,428 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T08:55:27,428 INFO [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66052323, jitterRate=-0.015743687748908997}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T08:55:27,430 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732265727402Initializing all the Stores at 1732265727402Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265727402Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265727403 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265727403Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265727403Cleaning up temporary data from old regions at 1732265727422 (+19 ms)Region opened successfully at 1732265727429 (+7 ms) 2024-11-22T08:55:27,430 INFO [master/296fbbacac79:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T08:55:27,434 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394b129f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=296fbbacac79/172.17.0.2:0 2024-11-22T08:55:27,435 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T08:55:27,435 INFO [master/296fbbacac79:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T08:55:27,435 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T08:55:27,436 INFO [master/296fbbacac79:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T08:55:27,436 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T08:55:27,436 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-22T08:55:27,436 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T08:55:27,438 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T08:55:27,439 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T08:55:27,440 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T08:55:27,440 INFO [master/296fbbacac79:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T08:55:27,441 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T08:55:27,442 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T08:55:27,442 INFO [master/296fbbacac79:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T08:55:27,443 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T08:55:27,444 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T08:55:27,445 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T08:55:27,446 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T08:55:27,448 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T08:55:27,449 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T08:55:27,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:27,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:27,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:27,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,451 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=296fbbacac79,39331,1732265727182, sessionid=0x10066a4b12e0000, setting cluster-up flag (Was=false) 2024-11-22T08:55:27,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,456 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T08:55:27,458 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=296fbbacac79,39331,1732265727182 2024-11-22T08:55:27,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,463 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T08:55:27,464 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=296fbbacac79,39331,1732265727182 2024-11-22T08:55:27,464 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T08:55:27,466 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T08:55:27,466 INFO [master/296fbbacac79:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T08:55:27,466 INFO [master/296fbbacac79:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T08:55:27,466 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 296fbbacac79,39331,1732265727182 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T08:55:27,468 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/296fbbacac79:0, corePoolSize=5, maxPoolSize=5 2024-11-22T08:55:27,468 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/296fbbacac79:0, corePoolSize=5, maxPoolSize=5 2024-11-22T08:55:27,468 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/296fbbacac79:0, corePoolSize=5, maxPoolSize=5 2024-11-22T08:55:27,468 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/296fbbacac79:0, corePoolSize=5, maxPoolSize=5 2024-11-22T08:55:27,468 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/296fbbacac79:0, corePoolSize=10, maxPoolSize=10 2024-11-22T08:55:27,469 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,469 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/296fbbacac79:0, corePoolSize=2, maxPoolSize=2 2024-11-22T08:55:27,469 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,470 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T08:55:27,470 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T08:55:27,471 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,471 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T08:55:27,471 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732265757471 2024-11-22T08:55:27,472 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T08:55:27,472 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T08:55:27,472 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T08:55:27,472 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T08:55:27,472 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T08:55:27,472 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T08:55:27,472 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,473 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T08:55:27,473 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T08:55:27,473 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T08:55:27,473 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T08:55:27,473 INFO [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T08:55:27,474 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.large.0-1732265727473,5,FailOnTimeoutGroup] 2024-11-22T08:55:27,477 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.small.0-1732265727474,5,FailOnTimeoutGroup] 2024-11-22T08:55:27,477 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,477 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T08:55:27,477 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,477 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,488 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(746): ClusterId : 05f9bea9-a0cc-4981-ae4a-0b352f94b7d9 2024-11-22T08:55:27,488 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(746): ClusterId : 05f9bea9-a0cc-4981-ae4a-0b352f94b7d9 2024-11-22T08:55:27,488 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T08:55:27,488 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T08:55:27,490 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T08:55:27,490 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T08:55:27,490 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T08:55:27,490 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T08:55:27,492 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T08:55:27,492 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T08:55:27,492 DEBUG [RS:1;296fbbacac79:41191 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ab67ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=296fbbacac79/172.17.0.2:0 2024-11-22T08:55:27,492 DEBUG [RS:0;296fbbacac79:34493 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3349ac58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=296fbbacac79/172.17.0.2:0 2024-11-22T08:55:27,494 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T08:55:27,494 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a 2024-11-22T08:55:27,506 DEBUG [RS:1;296fbbacac79:41191 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;296fbbacac79:41191 2024-11-22T08:55:27,507 INFO [RS:1;296fbbacac79:41191 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T08:55:27,507 INFO [RS:1;296fbbacac79:41191 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T08:55:27,507 DEBUG [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T08:55:27,508 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(2659): reportForDuty to master=296fbbacac79,39331,1732265727182 with port=41191, startcode=1732265727268 2024-11-22T08:55:27,508 DEBUG [RS:1;296fbbacac79:41191 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T08:55:27,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:27,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T08:55:27,512 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54979, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T08:55:27,512 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39331 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 296fbbacac79,41191,1732265727268 2024-11-22T08:55:27,512 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39331 {}] master.ServerManager(517): Registering regionserver=296fbbacac79,41191,1732265727268 2024-11-22T08:55:27,514 DEBUG [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a 2024-11-22T08:55:27,514 DEBUG [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T08:55:27,514 DEBUG [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T08:55:27,514 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T08:55:27,515 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:27,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T08:55:27,515 DEBUG [RS:0;296fbbacac79:34493 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;296fbbacac79:34493 2024-11-22T08:55:27,515 INFO [RS:0;296fbbacac79:34493 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T08:55:27,516 INFO [RS:0;296fbbacac79:34493 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T08:55:27,516 DEBUG [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T08:55:27,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T08:55:27,517 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(2659): reportForDuty to master=296fbbacac79,39331,1732265727182 with port=34493, startcode=1732265727234 2024-11-22T08:55:27,517 DEBUG [RS:1;296fbbacac79:41191 {}] zookeeper.ZKUtil(111): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/296fbbacac79,41191,1732265727268 2024-11-22T08:55:27,517 WARN [RS:1;296fbbacac79:41191 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T08:55:27,517 DEBUG [RS:0;296fbbacac79:34493 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T08:55:27,517 INFO [RS:1;296fbbacac79:41191 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T08:55:27,517 DEBUG [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,41191,1732265727268 2024-11-22T08:55:27,518 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [296fbbacac79,41191,1732265727268] 2024-11-22T08:55:27,522 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33957, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T08:55:27,523 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39331 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 296fbbacac79,34493,1732265727234 2024-11-22T08:55:27,523 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39331 {}] master.ServerManager(517): Registering regionserver=296fbbacac79,34493,1732265727234 2024-11-22T08:55:27,523 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T08:55:27,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,523 INFO [RS:1;296fbbacac79:41191 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T08:55:27,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:27,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T08:55:27,524 DEBUG [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a 2024-11-22T08:55:27,524 DEBUG [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T08:55:27,524 DEBUG [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T08:55:27,526 INFO [RS:1;296fbbacac79:41191 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T08:55:27,527 INFO [RS:1;296fbbacac79:41191 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T08:55:27,527 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T08:55:27,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T08:55:27,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:27,528 DEBUG [RS:0;296fbbacac79:34493 {}] zookeeper.ZKUtil(111): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/296fbbacac79,34493,1732265727234 2024-11-22T08:55:27,528 WARN [RS:0;296fbbacac79:34493 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T08:55:27,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T08:55:27,528 INFO [RS:0;296fbbacac79:34493 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T08:55:27,528 DEBUG [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,34493,1732265727234 2024-11-22T08:55:27,528 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [296fbbacac79,34493,1732265727234] 2024-11-22T08:55:27,533 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T08:55:27,534 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T08:55:27,534 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,534 INFO [RS:1;296fbbacac79:41191 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T08:55:27,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:27,534 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T08:55:27,534 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,534 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,535 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,535 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,535 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740 2024-11-22T08:55:27,535 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/296fbbacac79:0, corePoolSize=2, maxPoolSize=2 2024-11-22T08:55:27,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740 2024-11-22T08:55:27,535 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,535 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T08:55:27,535 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T08:55:27,535 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,536 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,536 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T08:55:27,536 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,536 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/296fbbacac79:0, corePoolSize=3, maxPoolSize=3 2024-11-22T08:55:27,536 DEBUG [RS:1;296fbbacac79:41191 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/296fbbacac79:0, corePoolSize=3, maxPoolSize=3 2024-11-22T08:55:27,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T08:55:27,536 INFO [RS:0;296fbbacac79:34493 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T08:55:27,537 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,537 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,537 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,537 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,537 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,537 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,41191,1732265727268-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T08:55:27,541 INFO [RS:0;296fbbacac79:34493 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T08:55:27,542 INFO [RS:0;296fbbacac79:34493 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T08:55:27,542 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,543 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T08:55:27,544 INFO [RS:0;296fbbacac79:34493 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T08:55:27,544 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,544 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,544 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,544 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,545 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,545 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,545 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/296fbbacac79:0, corePoolSize=2, maxPoolSize=2 2024-11-22T08:55:27,545 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,545 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,545 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,545 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,545 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,546 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:27,546 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/296fbbacac79:0, corePoolSize=3, maxPoolSize=3 2024-11-22T08:55:27,546 DEBUG [RS:0;296fbbacac79:34493 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/296fbbacac79:0, corePoolSize=3, maxPoolSize=3 2024-11-22T08:55:27,546 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T08:55:27,546 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,546 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,546 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,546 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75233492, jitterRate=0.1210663914680481}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T08:55:27,547 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,547 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,547 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,34493,1732265727234-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T08:55:27,547 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732265727509Initializing all the Stores at 1732265727509Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265727509Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265727509Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265727509Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265727509Cleaning up temporary data from old regions at 1732265727535 (+26 ms)Region opened successfully at 1732265727547 (+12 ms) 2024-11-22T08:55:27,547 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T08:55:27,547 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T08:55:27,547 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T08:55:27,547 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T08:55:27,547 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T08:55:27,549 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T08:55:27,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732265727547Disabling compacts and flushes for region at 1732265727547Disabling writes for close at 1732265727547Writing region close event to WAL at 1732265727549 (+2 ms)Closed at 1732265727549 2024-11-22T08:55:27,551 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T08:55:27,551 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T08:55:27,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T08:55:27,552 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T08:55:27,553 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T08:55:27,555 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T08:55:27,556 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,41191,1732265727268-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,556 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,556 INFO [RS:1;296fbbacac79:41191 {}] regionserver.Replication(171): 296fbbacac79,41191,1732265727268 started 2024-11-22T08:55:27,569 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T08:55:27,569 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,34493,1732265727234-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,569 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,569 INFO [RS:0;296fbbacac79:34493 {}] regionserver.Replication(171): 296fbbacac79,34493,1732265727234 started 2024-11-22T08:55:27,571 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,571 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(1482): Serving as 296fbbacac79,41191,1732265727268, RpcServer on 296fbbacac79/172.17.0.2:41191, sessionid=0x10066a4b12e0002 2024-11-22T08:55:27,571 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T08:55:27,571 DEBUG [RS:1;296fbbacac79:41191 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 296fbbacac79,41191,1732265727268 2024-11-22T08:55:27,571 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '296fbbacac79,41191,1732265727268' 2024-11-22T08:55:27,571 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T08:55:27,572 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T08:55:27,572 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T08:55:27,572 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T08:55:27,572 DEBUG [RS:1;296fbbacac79:41191 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 296fbbacac79,41191,1732265727268 2024-11-22T08:55:27,572 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '296fbbacac79,41191,1732265727268' 2024-11-22T08:55:27,572 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T08:55:27,573 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T08:55:27,573 DEBUG [RS:1;296fbbacac79:41191 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T08:55:27,573 INFO [RS:1;296fbbacac79:41191 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T08:55:27,573 INFO [RS:1;296fbbacac79:41191 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T08:55:27,591 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,591 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(1482): Serving as 296fbbacac79,34493,1732265727234, RpcServer on 296fbbacac79/172.17.0.2:34493, sessionid=0x10066a4b12e0001 2024-11-22T08:55:27,591 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T08:55:27,591 DEBUG [RS:0;296fbbacac79:34493 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 296fbbacac79,34493,1732265727234 2024-11-22T08:55:27,591 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '296fbbacac79,34493,1732265727234' 2024-11-22T08:55:27,592 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T08:55:27,592 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T08:55:27,593 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T08:55:27,593 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T08:55:27,593 DEBUG [RS:0;296fbbacac79:34493 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 296fbbacac79,34493,1732265727234 2024-11-22T08:55:27,593 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '296fbbacac79,34493,1732265727234' 2024-11-22T08:55:27,593 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T08:55:27,594 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T08:55:27,594 DEBUG [RS:0;296fbbacac79:34493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T08:55:27,594 INFO [RS:0;296fbbacac79:34493 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T08:55:27,594 INFO [RS:0;296fbbacac79:34493 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T08:55:27,675 INFO [RS:1;296fbbacac79:41191 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T08:55:27,676 INFO [RS:1;296fbbacac79:41191 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=296fbbacac79%2C41191%2C1732265727268, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,41191,1732265727268, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs, maxLogs=55 2024-11-22T08:55:27,695 INFO [RS:0;296fbbacac79:34493 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T08:55:27,696 INFO [RS:0;296fbbacac79:34493 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=296fbbacac79%2C34493%2C1732265727234, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,34493,1732265727234, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs, maxLogs=55 2024-11-22T08:55:27,699 INFO [RS:1;296fbbacac79:41191 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,41191,1732265727268/296fbbacac79%2C41191%2C1732265727268.1732265727680 2024-11-22T08:55:27,699 DEBUG [RS:1;296fbbacac79:41191 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T08:55:27,703 DEBUG [296fbbacac79:39331 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T08:55:27,704 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=296fbbacac79,41191,1732265727268 2024-11-22T08:55:27,705 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 296fbbacac79,41191,1732265727268, state=OPENING 2024-11-22T08:55:27,706 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T08:55:27,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:27,708 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T08:55:27,708 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:27,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=296fbbacac79,41191,1732265727268}] 2024-11-22T08:55:27,709 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:27,709 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:27,718 INFO [RS:0;296fbbacac79:34493 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,34493,1732265727234/296fbbacac79%2C34493%2C1732265727234.1732265727697 2024-11-22T08:55:27,718 DEBUG [RS:0;296fbbacac79:34493 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T08:55:27,863 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T08:55:27,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59235, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T08:55:27,873 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T08:55:27,873 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T08:55:27,874 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T08:55:27,875 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=296fbbacac79%2C41191%2C1732265727268.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,41191,1732265727268, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs, maxLogs=55 2024-11-22T08:55:27,892 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,41191,1732265727268/296fbbacac79%2C41191%2C1732265727268.meta.1732265727876.meta 2024-11-22T08:55:27,893 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T08:55:27,893 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T08:55:27,893 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T08:55:27,893 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T08:55:27,894 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T08:55:27,894 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T08:55:27,894 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:27,894 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T08:55:27,894 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T08:55:27,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T08:55:27,895 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T08:55:27,896 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:27,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T08:55:27,896 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T08:55:27,896 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:27,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T08:55:27,897 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T08:55:27,897 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:27,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T08:55:27,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T08:55:27,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:27,898 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:27,898 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T08:55:27,898 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740 2024-11-22T08:55:27,899 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740 2024-11-22T08:55:27,899 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T08:55:27,899 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T08:55:27,899 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T08:55:27,900 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T08:55:27,900 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64815113, jitterRate=-0.034179553389549255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T08:55:27,900 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T08:55:27,901 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732265727894Writing region info on filesystem at 1732265727894Initializing all the Stores at 1732265727894Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265727895 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265727895Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265727895Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265727895Cleaning up temporary data from old regions at 1732265727899 (+4 ms)Running coprocessor post-open hooks at 1732265727900 (+1 ms)Region opened successfully at 1732265727901 (+1 ms) 2024-11-22T08:55:27,901 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732265727862 2024-11-22T08:55:27,904 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T08:55:27,904 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T08:55:27,904 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=296fbbacac79,41191,1732265727268 2024-11-22T08:55:27,905 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 296fbbacac79,41191,1732265727268, state=OPEN 2024-11-22T08:55:27,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:27,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:27,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:27,906 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=296fbbacac79,41191,1732265727268 2024-11-22T08:55:27,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:27,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:27,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:27,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T08:55:27,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=296fbbacac79,41191,1732265727268 in 198 msec 2024-11-22T08:55:27,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T08:55:27,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 358 msec 2024-11-22T08:55:27,911 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T08:55:27,911 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T08:55:27,914 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T08:55:27,914 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=296fbbacac79,41191,1732265727268, seqNum=-1] 2024-11-22T08:55:27,915 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T08:55:27,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44369, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T08:55:27,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 456 msec 2024-11-22T08:55:27,923 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732265727923, completionTime=-1 2024-11-22T08:55:27,923 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-22T08:55:27,923 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T08:55:27,932 INFO [master/296fbbacac79:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T08:55:27,932 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732265787932 2024-11-22T08:55:27,932 INFO [master/296fbbacac79:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732265847932 2024-11-22T08:55:27,932 INFO [master/296fbbacac79:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-22T08:55:27,933 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,39331,1732265727182-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,934 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,39331,1732265727182-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,934 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,39331,1732265727182-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,934 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-296fbbacac79:39331, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,934 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,935 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,939 DEBUG [master/296fbbacac79:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T08:55:27,943 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.637sec 2024-11-22T08:55:27,944 INFO [master/296fbbacac79:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T08:55:27,944 INFO [master/296fbbacac79:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T08:55:27,944 INFO [master/296fbbacac79:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T08:55:27,944 INFO [master/296fbbacac79:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T08:55:27,944 INFO [master/296fbbacac79:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T08:55:27,944 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,39331,1732265727182-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T08:55:27,944 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,39331,1732265727182-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T08:55:27,945 DEBUG [master/296fbbacac79:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T08:55:27,945 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T08:55:27,946 INFO [master/296fbbacac79:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,39331,1732265727182-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:27,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T08:55:27,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 296fbbacac79,39331,-1 for getting cluster id 2024-11-22T08:55:27,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T08:55:27,990 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '05f9bea9-a0cc-4981-ae4a-0b352f94b7d9' 2024-11-22T08:55:27,991 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T08:55:27,991 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "05f9bea9-a0cc-4981-ae4a-0b352f94b7d9" 2024-11-22T08:55:27,991 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b98229c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T08:55:27,991 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [296fbbacac79,39331,-1] 2024-11-22T08:55:27,992 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T08:55:27,992 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:27,994 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T08:55:27,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T08:55:27,995 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T08:55:27,997 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=296fbbacac79,41191,1732265727268, seqNum=-1] 2024-11-22T08:55:27,997 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T08:55:28,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T08:55:28,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=296fbbacac79,39331,1732265727182 2024-11-22T08:55:28,002 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T08:55:28,004 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 296fbbacac79,39331,1732265727182 2024-11-22T08:55:28,004 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@424256dd 2024-11-22T08:55:28,004 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T08:55:28,006 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T08:55:28,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39331 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732265728002', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T08:55:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39331 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732265728002 2024-11-22T08:55:28,010 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732265728002 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T08:55:28,010 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:28,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39331 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732265728002" procId is: 4 2024-11-22T08:55:28,012 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732265728002 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T08:55:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T08:55:28,025 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732265728002-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => db7cb69e73f1bcaf57618c7b6060702d, NAME => 'testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732265728002', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a 2024-11-22T08:55:28,031 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732265728002-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:28,031 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732265728002-pool-0 {}] regionserver.HRegion(1722): Closing db7cb69e73f1bcaf57618c7b6060702d, disabling compactions & flushes 2024-11-22T08:55:28,031 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732265728002-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:28,031 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732265728002-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:28,031 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732265728002-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. after waiting 0 ms 2024-11-22T08:55:28,031 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732265728002-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:28,031 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732265728002-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:28,031 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732265728002-pool-0 {}] regionserver.HRegion(1676): Region close journal for db7cb69e73f1bcaf57618c7b6060702d: Waiting for close lock at 1732265728031Disabling compacts and flushes for region at 1732265728031Disabling writes for close at 1732265728031Writing region close event to WAL at 1732265728031Closed at 1732265728031 2024-11-22T08:55:28,033 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732265728002 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T08:55:28,033 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732265728033"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732265728033"}]},"ts":"1732265728033"} 2024-11-22T08:55:28,036 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T08:55:28,037 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732265728002 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T08:55:28,038 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732265728002","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732265728038"}]},"ts":"1732265728038"} 2024-11-22T08:55:28,040 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732265728002, state=ENABLING in hbase:meta 2024-11-22T08:55:28,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {296fbbacac79=0} racks are {/default-rack=0} 2024-11-22T08:55:28,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T08:55:28,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T08:55:28,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T08:55:28,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T08:55:28,041 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T08:55:28,041 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T08:55:28,041 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T08:55:28,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732265728002, region=db7cb69e73f1bcaf57618c7b6060702d, ASSIGN}] 2024-11-22T08:55:28,043 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732265728002, region=db7cb69e73f1bcaf57618c7b6060702d, ASSIGN 2024-11-22T08:55:28,044 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732265728002, region=db7cb69e73f1bcaf57618c7b6060702d, ASSIGN; state=OFFLINE, location=296fbbacac79,41191,1732265727268; forceNewPlan=false, retain=false 2024-11-22T08:55:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T08:55:28,195 INFO [296fbbacac79:39331 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T08:55:28,195 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=db7cb69e73f1bcaf57618c7b6060702d, regionState=OPENING, regionLocation=296fbbacac79,41191,1732265727268 2024-11-22T08:55:28,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732265728002, region=db7cb69e73f1bcaf57618c7b6060702d, ASSIGN because future has completed 2024-11-22T08:55:28,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure db7cb69e73f1bcaf57618c7b6060702d, server=296fbbacac79,41191,1732265727268}] 2024-11-22T08:55:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T08:55:28,354 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:28,354 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => db7cb69e73f1bcaf57618c7b6060702d, NAME => 'testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d.', STARTKEY => '', ENDKEY => ''} 2024-11-22T08:55:28,355 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732265728002 db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,355 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:28,355 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,355 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,356 INFO [StoreOpener-db7cb69e73f1bcaf57618c7b6060702d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,359 INFO [StoreOpener-db7cb69e73f1bcaf57618c7b6060702d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db7cb69e73f1bcaf57618c7b6060702d columnFamilyName family 2024-11-22T08:55:28,359 DEBUG [StoreOpener-db7cb69e73f1bcaf57618c7b6060702d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:28,360 INFO [StoreOpener-db7cb69e73f1bcaf57618c7b6060702d-1 {}] regionserver.HStore(327): Store=db7cb69e73f1bcaf57618c7b6060702d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:28,360 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,361 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,361 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,361 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,361 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,362 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,367 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T08:55:28,368 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened db7cb69e73f1bcaf57618c7b6060702d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66413643, jitterRate=-0.010359600186347961}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T08:55:28,368 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:28,368 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for db7cb69e73f1bcaf57618c7b6060702d: Running coprocessor pre-open hook at 1732265728355Writing region info on filesystem at 1732265728355Initializing all the Stores at 1732265728355Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265728355Cleaning up temporary data from old regions at 1732265728361 (+6 ms)Running coprocessor post-open hooks at 1732265728368 (+7 ms)Region opened successfully at 1732265728368 2024-11-22T08:55:28,369 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d., pid=6, masterSystemTime=1732265728351 2024-11-22T08:55:28,371 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:28,372 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:28,372 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=db7cb69e73f1bcaf57618c7b6060702d, regionState=OPEN, openSeqNum=2, regionLocation=296fbbacac79,41191,1732265727268 2024-11-22T08:55:28,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure db7cb69e73f1bcaf57618c7b6060702d, server=296fbbacac79,41191,1732265727268 because future has completed 2024-11-22T08:55:28,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T08:55:28,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure db7cb69e73f1bcaf57618c7b6060702d, server=296fbbacac79,41191,1732265727268 in 177 msec 2024-11-22T08:55:28,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T08:55:28,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732265728002, region=db7cb69e73f1bcaf57618c7b6060702d, ASSIGN in 336 msec 2024-11-22T08:55:28,381 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732265728002 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T08:55:28,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732265728002","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732265728381"}]},"ts":"1732265728381"} 2024-11-22T08:55:28,384 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732265728002, state=ENABLED in hbase:meta 2024-11-22T08:55:28,385 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732265728002 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T08:55:28,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732265728002 in 378 msec 2024-11-22T08:55:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39331 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T08:55:28,649 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732265728002 completed 2024-11-22T08:55:28,649 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732265728002 get assigned. Timeout = 60000ms 2024-11-22T08:55:28,650 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T08:55:28,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732265728002 assigned to meta. Checking AM states. 2024-11-22T08:55:28,656 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T08:55:28,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732265728002 assigned. 2024-11-22T08:55:28,661 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732265728002', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d., hostname=296fbbacac79,41191,1732265727268, seqNum=2] 2024-11-22T08:55:28,667 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing db7cb69e73f1bcaf57618c7b6060702d 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T08:55:28,674 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 276d2fff1875424cafc3768aa6dee2b7 2024-11-22T08:55:28,689 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/.tmp/family/276d2fff1875424cafc3768aa6dee2b7 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T08:55:28,689 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/.tmp/family/276d2fff1875424cafc3768aa6dee2b7 2024-11-22T08:55:28,692 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/.tmp/family/276d2fff1875424cafc3768aa6dee2b7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7 2024-11-22T08:55:28,694 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7, delay=986 ms 2024-11-22T08:55:28,695 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T08:55:28,696 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for db7cb69e73f1bcaf57618c7b6060702d in 29ms, sequenceid=6, compaction requested=false 2024-11-22T08:55:28,696 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732265728002' 2024-11-22T08:55:28,697 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for db7cb69e73f1bcaf57618c7b6060702d: 2024-11-22T08:55:29,529 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T08:55:29,681 DEBUG [hfile-prefetch-1732265726648 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 276d2fff1875424cafc3768aa6dee2b7_0. 2024-11-22T08:55:29,682 DEBUG [hfile-prefetch-1732265726648 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 276d2fff1875424cafc3768aa6dee2b7_117. 2024-11-22T08:55:29,682 DEBUG [hfile-prefetch-1732265726648 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7, with total blocks 2, and data blocks 1 2024-11-22T08:55:29,682 DEBUG [hfile-prefetch-1732265726648 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T08:55:29,683 DEBUG [hfile-prefetch-1732265726648 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T08:55:29,683 DEBUG [hfile-prefetch-1732265726648 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7. Will sleep 100ms and try the verification again. 2024-11-22T08:55:29,697 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;296fbbacac79:41191,5,FailOnTimeoutGroup] 2024-11-22T08:55:29,698 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '296fbbacac79,41191,1732265727268' ***** 2024-11-22T08:55:29,698 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-22T08:55:29,698 INFO [RS:1;296fbbacac79:41191 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T08:55:29,698 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T08:55:29,699 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T08:55:29,699 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T08:55:29,699 INFO [RS:1;296fbbacac79:41191 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.persistence 2024-11-22T08:55:29,699 INFO [RS:1;296fbbacac79:41191 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T08:55:29,699 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T08:55:29,709 DEBUG [RS:1;296fbbacac79:41191 {}] bucket.BucketCache(1487): Thread RS:1;296fbbacac79:41191 started persisting bucket cache to file 2024-11-22T08:55:29,709 DEBUG [RS:1;296fbbacac79:41191 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T08:55:29,709 DEBUG [RS:1;296fbbacac79:41191 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T08:55:29,715 DEBUG [RS:1;296fbbacac79:41191 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T08:55:29,715 DEBUG [RS:1;296fbbacac79:41191 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732265729709 2024-11-22T08:55:29,716 DEBUG [RS:1;296fbbacac79:41191 {}] bucket.BucketCache(1509): Thread RS:1;296fbbacac79:41191 finished persisting bucket cache to file, renaming 2024-11-22T08:55:29,716 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T08:55:29,716 INFO [RS:1;296fbbacac79:41191 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T08:55:29,716 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T08:55:29,716 INFO [RS:1;296fbbacac79:41191 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T08:55:29,716 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(3091): Received CLOSE for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:29,717 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(959): stopping server 296fbbacac79,41191,1732265727268 2024-11-22T08:55:29,717 INFO [RS:1;296fbbacac79:41191 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T08:55:29,717 INFO [RS:1;296fbbacac79:41191 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;296fbbacac79:41191. 2024-11-22T08:55:29,717 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T08:55:29,718 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing db7cb69e73f1bcaf57618c7b6060702d, disabling compactions & flushes 2024-11-22T08:55:29,718 DEBUG [RS:1;296fbbacac79:41191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T08:55:29,718 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:29,718 DEBUG [RS:1;296fbbacac79:41191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:29,718 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:29,718 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. after waiting 0 ms 2024-11-22T08:55:29,718 INFO [RS:1;296fbbacac79:41191 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T08:55:29,719 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:29,719 INFO [RS:1;296fbbacac79:41191 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T08:55:29,719 INFO [RS:1;296fbbacac79:41191 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T08:55:29,719 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T08:55:29,720 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-22T08:55:29,720 DEBUG [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, db7cb69e73f1bcaf57618c7b6060702d=testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d.} 2024-11-22T08:55:29,720 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T08:55:29,720 DEBUG [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:29,720 DEBUG [StoreFileCloser-db7cb69e73f1bcaf57618c7b6060702d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7 2024-11-22T08:55:29,720 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T08:55:29,720 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T08:55:29,721 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T08:55:29,721 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T08:55:29,720 WARN [hfile-prefetch-1732265726648 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-22T08:55:29,721 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T08:55:29,721 DEBUG [hfile-prefetch-1732265726648 {}] hfile.PrefetchExecutor(121): Prefetch completed for 276d2fff1875424cafc3768aa6dee2b7 2024-11-22T08:55:29,722 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-22T08:55:29,727 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T08:55:29,727 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:29,728 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for db7cb69e73f1bcaf57618c7b6060702d: Waiting for close lock at 1732265729717Running coprocessor pre-close hooks at 1732265729718 (+1 ms)Disabling compacts and flushes for region at 1732265729718Disabling writes for close at 1732265729718Writing region close event to WAL at 1732265729721 (+3 ms)Running coprocessor post-close hooks at 1732265729727 (+6 ms)Closed at 1732265729727 2024-11-22T08:55:29,728 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:29,735 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 262f5888ee854004a70697ea0a90d66c 2024-11-22T08:55:29,740 INFO [regionserver/296fbbacac79:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T08:55:29,744 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T08:55:29,744 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T08:55:29,744 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T08:55:29,744 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T08:55:29,747 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T08:55:29,747 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732265729744 2024-11-22T08:55:29,747 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T08:55:29,748 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/info/262f5888ee854004a70697ea0a90d66c is 209, key is testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d./info:regioninfo/1732265728372/Put/seqid=0 2024-11-22T08:55:29,749 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/info/262f5888ee854004a70697ea0a90d66c 2024-11-22T08:55:29,756 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2c16a3026c6f4944957680bd2bbd487c 2024-11-22T08:55:29,768 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/ns/2c16a3026c6f4944957680bd2bbd487c is 43, key is default/ns:d/1732265727917/Put/seqid=0 2024-11-22T08:55:29,769 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/ns/2c16a3026c6f4944957680bd2bbd487c 2024-11-22T08:55:29,777 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b23bf70213a64c9e92a106b0d1c6e784 2024-11-22T08:55:29,789 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/table/b23bf70213a64c9e92a106b0d1c6e784 is 80, key is testBlockEvictionOnGracefulStop1732265728002/table:state/1732265728381/Put/seqid=0 2024-11-22T08:55:29,790 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/table/b23bf70213a64c9e92a106b0d1c6e784 2024-11-22T08:55:29,792 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/info/262f5888ee854004a70697ea0a90d66c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/262f5888ee854004a70697ea0a90d66c 2024-11-22T08:55:29,794 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/262f5888ee854004a70697ea0a90d66c, delay=942 ms 2024-11-22T08:55:29,795 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/262f5888ee854004a70697ea0a90d66c, entries=10, sequenceid=11, filesize=7.0 K 2024-11-22T08:55:29,795 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/ns/2c16a3026c6f4944957680bd2bbd487c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/2c16a3026c6f4944957680bd2bbd487c 2024-11-22T08:55:29,797 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/2c16a3026c6f4944957680bd2bbd487c, delay=997 ms 2024-11-22T08:55:29,798 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/2c16a3026c6f4944957680bd2bbd487c, entries=2, sequenceid=11, filesize=5.0 K 2024-11-22T08:55:29,798 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/table/b23bf70213a64c9e92a106b0d1c6e784 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/b23bf70213a64c9e92a106b0d1c6e784 2024-11-22T08:55:29,799 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/b23bf70213a64c9e92a106b0d1c6e784, delay=964 ms 2024-11-22T08:55:29,800 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/b23bf70213a64c9e92a106b0d1c6e784, entries=2, sequenceid=11, filesize=5.3 K 2024-11-22T08:55:29,801 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=11, compaction requested=false 2024-11-22T08:55:29,802 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/262f5888ee854004a70697ea0a90d66c 2024-11-22T08:55:29,802 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/2c16a3026c6f4944957680bd2bbd487c 2024-11-22T08:55:29,803 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/b23bf70213a64c9e92a106b0d1c6e784 2024-11-22T08:55:29,807 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-22T08:55:29,807 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T08:55:29,807 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T08:55:29,808 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732265729720Running coprocessor pre-close hooks at 1732265729720Disabling compacts and flushes for region at 1732265729720Disabling writes for close at 1732265729721 (+1 ms)Obtaining lock to block concurrent updates at 1732265729722 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732265729722Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732265729723 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732265729723Flushing 1588230740/info: creating writer at 1732265729723Flushing 1588230740/info: appending metadata at 1732265729748 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732265729748Flushing 1588230740/ns: creating writer at 1732265729751 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732265729767 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732265729767Flushing 1588230740/table: creating writer at 1732265729771 (+4 ms)Flushing 1588230740/table: appending metadata at 1732265729789 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732265729789Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e898ed4: reopening flushed file at 1732265729792 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38d8c96d: reopening flushed file at 1732265729795 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64e4b332: reopening flushed file at 1732265729798 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=11, compaction requested=false at 1732265729801 (+3 ms)Writing region close event to WAL at 1732265729803 (+2 ms)Running coprocessor post-close hooks at 1732265729807 (+4 ms)Closed at 1732265729807 2024-11-22T08:55:29,808 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T08:55:29,920 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(976): stopping server 296fbbacac79,41191,1732265727268; all regions closed. 2024-11-22T08:55:29,923 DEBUG [RS:1;296fbbacac79:41191 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs 2024-11-22T08:55:29,923 INFO [RS:1;296fbbacac79:41191 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 296fbbacac79%2C41191%2C1732265727268.meta:.meta(num 1732265727876) 2024-11-22T08:55:29,926 DEBUG [RS:1;296fbbacac79:41191 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs 2024-11-22T08:55:29,926 INFO [RS:1;296fbbacac79:41191 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 296fbbacac79%2C41191%2C1732265727268:(num 1732265727680) 2024-11-22T08:55:29,926 DEBUG [RS:1;296fbbacac79:41191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:29,926 INFO [RS:1;296fbbacac79:41191 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T08:55:29,926 INFO [RS:1;296fbbacac79:41191 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T08:55:29,926 INFO [RS:1;296fbbacac79:41191 {}] hbase.ChoreService(370): Chore service for: regionserver/296fbbacac79:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T08:55:29,927 INFO [RS:1;296fbbacac79:41191 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T08:55:29,927 INFO [regionserver/296fbbacac79:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T08:55:29,927 INFO [RS:1;296fbbacac79:41191 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41191 2024-11-22T08:55:29,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T08:55:29,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/296fbbacac79,41191,1732265727268 2024-11-22T08:55:29,931 INFO [RS:1;296fbbacac79:41191 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T08:55:29,932 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [296fbbacac79,41191,1732265727268] 2024-11-22T08:55:29,933 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/296fbbacac79,41191,1732265727268 already deleted, retry=false 2024-11-22T08:55:29,933 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 296fbbacac79,41191,1732265727268 on 296fbbacac79,39331,1732265727182 2024-11-22T08:55:29,937 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 296fbbacac79,41191,1732265727268, splitWal=true, meta=true 2024-11-22T08:55:29,939 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 296fbbacac79,41191,1732265727268 (carryingMeta=true) 296fbbacac79,41191,1732265727268/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@5de789cb[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-22T08:55:29,939 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 296fbbacac79,41191,1732265727268, splitWal=true, meta=true 2024-11-22T08:55:29,940 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 296fbbacac79,41191,1732265727268, splitWal=true, meta=true, isMeta: true 2024-11-22T08:55:29,941 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,41191,1732265727268-splitting 2024-11-22T08:55:29,941 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,41191,1732265727268-splitting dir is empty, no logs to split. 2024-11-22T08:55:29,941 INFO [PEWorker-2 {}] master.SplitWALManager(105): 296fbbacac79,41191,1732265727268 WAL count=0, meta=true 2024-11-22T08:55:29,943 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,41191,1732265727268-splitting dir is empty, no logs to split. 2024-11-22T08:55:29,943 INFO [PEWorker-2 {}] master.SplitWALManager(105): 296fbbacac79,41191,1732265727268 WAL count=0, meta=true 2024-11-22T08:55:29,943 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 296fbbacac79,41191,1732265727268 WAL splitting is done? wals=0, meta=true 2024-11-22T08:55:29,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T08:55:29,945 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T08:55:29,946 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T08:55:30,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:30,033 INFO [RS:1;296fbbacac79:41191 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T08:55:30,033 INFO [RS:1;296fbbacac79:41191 {}] regionserver.HRegionServer(1031): Exiting; stopping=296fbbacac79,41191,1732265727268; zookeeper connection closed. 2024-11-22T08:55:30,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41191-0x10066a4b12e0002, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:30,033 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3229b60f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3229b60f 2024-11-22T08:55:30,096 DEBUG [296fbbacac79:39331 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T08:55:30,098 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=296fbbacac79,34493,1732265727234 2024-11-22T08:55:30,098 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 296fbbacac79,34493,1732265727234, state=OPENING 2024-11-22T08:55:30,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:30,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:30,100 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T08:55:30,100 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:30,100 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:30,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=296fbbacac79,34493,1732265727234}] 2024-11-22T08:55:30,218 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/296fbbacac79:0 server-side Connection retries=45 2024-11-22T08:55:30,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:30,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:30,219 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T08:55:30,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T08:55:30,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T08:55:30,219 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T08:55:30,219 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T08:55:30,220 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36553 2024-11-22T08:55:30,221 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36553 connecting to ZooKeeper ensemble=127.0.0.1:53512 2024-11-22T08:55:30,221 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:30,222 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:30,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365530x0, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T08:55:30,225 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:365530x0, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-22T08:55:30,225 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-22T08:55:30,225 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36553-0x10066a4b12e0003 connected 2024-11-22T08:55:30,226 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T08:55:30,226 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.cache 2024-11-22T08:55:30,226 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T08:55:30,226 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T08:55:30,226 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T08:55:30,228 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T08:55:30,228 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T08:55:30,228 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T08:55:30,229 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-22T08:55:30,229 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36553-0x10066a4b12e0003, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T08:55:30,229 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-22T08:55:30,231 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36553-0x10066a4b12e0003, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T08:55:30,235 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36553 2024-11-22T08:55:30,236 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-22T08:55:30,236 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T08:55:30,237 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-22T08:55:30,237 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.persistence is complete. 2024-11-22T08:55:30,237 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36553 2024-11-22T08:55:30,237 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36553 2024-11-22T08:55:30,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36553 2024-11-22T08:55:30,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36553 2024-11-22T08:55:30,243 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(746): ClusterId : 05f9bea9-a0cc-4981-ae4a-0b352f94b7d9 2024-11-22T08:55:30,243 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T08:55:30,245 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T08:55:30,245 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T08:55:30,247 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T08:55:30,247 DEBUG [RS:2;296fbbacac79:36553 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295e6117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=296fbbacac79/172.17.0.2:0 2024-11-22T08:55:30,252 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T08:55:30,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37427, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T08:55:30,256 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T08:55:30,256 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T08:55:30,257 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T08:55:30,257 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=296fbbacac79%2C34493%2C1732265727234.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,34493,1732265727234, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs, maxLogs=55 2024-11-22T08:55:30,260 DEBUG [RS:2;296fbbacac79:36553 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;296fbbacac79:36553 2024-11-22T08:55:30,260 INFO [RS:2;296fbbacac79:36553 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T08:55:30,260 INFO [RS:2;296fbbacac79:36553 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T08:55:30,260 DEBUG [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T08:55:30,261 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(2659): reportForDuty to master=296fbbacac79,39331,1732265727182 with port=36553, startcode=1732265730218 2024-11-22T08:55:30,261 DEBUG [RS:2;296fbbacac79:36553 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T08:55:30,263 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47783, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T08:55:30,264 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39331 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 296fbbacac79,36553,1732265730218 2024-11-22T08:55:30,264 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39331 {}] master.ServerManager(517): Registering regionserver=296fbbacac79,36553,1732265730218 2024-11-22T08:55:30,265 DEBUG [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a 2024-11-22T08:55:30,265 DEBUG [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T08:55:30,265 DEBUG [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T08:55:30,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T08:55:30,267 DEBUG [RS:2;296fbbacac79:36553 {}] zookeeper.ZKUtil(111): regionserver:36553-0x10066a4b12e0003, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/296fbbacac79,36553,1732265730218 2024-11-22T08:55:30,268 WARN [RS:2;296fbbacac79:36553 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T08:55:30,268 INFO [RS:2;296fbbacac79:36553 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T08:55:30,268 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [296fbbacac79,36553,1732265730218] 2024-11-22T08:55:30,268 DEBUG [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,36553,1732265730218 2024-11-22T08:55:30,275 INFO [RS:2;296fbbacac79:36553 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T08:55:30,279 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,34493,1732265727234/296fbbacac79%2C34493%2C1732265727234.meta.1732265730258.meta 2024-11-22T08:55:30,280 INFO [RS:2;296fbbacac79:36553 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T08:55:30,280 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T08:55:30,281 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T08:55:30,281 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T08:55:30,281 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T08:55:30,281 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T08:55:30,281 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T08:55:30,282 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:30,282 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T08:55:30,282 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T08:55:30,284 INFO [RS:2;296fbbacac79:36553 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T08:55:30,284 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:30,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T08:55:30,285 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T08:55:30,285 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T08:55:30,285 INFO [RS:2;296fbbacac79:36553 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T08:55:30,285 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:30,285 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/296fbbacac79:0, corePoolSize=2, maxPoolSize=2 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/296fbbacac79:0, corePoolSize=1, maxPoolSize=1 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/296fbbacac79:0, corePoolSize=3, maxPoolSize=3 2024-11-22T08:55:30,286 DEBUG [RS:2;296fbbacac79:36553 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/296fbbacac79:0, corePoolSize=3, maxPoolSize=3 2024-11-22T08:55:30,288 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:30,288 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/262f5888ee854004a70697ea0a90d66c, delay=997 ms 2024-11-22T08:55:30,288 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:30,289 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:30,289 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:30,289 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:30,289 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,36553,1732265730218-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T08:55:30,289 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/262f5888ee854004a70697ea0a90d66c 2024-11-22T08:55:30,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:30,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T08:55:30,290 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T08:55:30,290 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:30,292 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/2c16a3026c6f4944957680bd2bbd487c, delay=980 ms 2024-11-22T08:55:30,294 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/2c16a3026c6f4944957680bd2bbd487c 2024-11-22T08:55:30,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:30,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T08:55:30,294 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T08:55:30,295 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:30,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:30,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T08:55:30,295 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T08:55:30,295 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:30,297 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/b23bf70213a64c9e92a106b0d1c6e784, delay=917 ms 2024-11-22T08:55:30,298 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/b23bf70213a64c9e92a106b0d1c6e784 2024-11-22T08:55:30,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T08:55:30,299 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T08:55:30,300 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740 2024-11-22T08:55:30,300 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740 2024-11-22T08:55:30,301 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T08:55:30,301 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T08:55:30,301 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T08:55:30,302 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T08:55:30,303 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63897437, jitterRate=-0.0478539913892746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T08:55:30,303 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T08:55:30,304 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732265730282Writing region info on filesystem at 1732265730282Initializing all the Stores at 1732265730282Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265730282Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265730284 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265730284Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732265730284Cleaning up temporary data from old regions at 1732265730301 (+17 ms)Running coprocessor post-open hooks at 1732265730303 (+2 ms)Region opened successfully at 1732265730304 (+1 ms) 2024-11-22T08:55:30,305 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732265730252 2024-11-22T08:55:30,306 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T08:55:30,306 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=296fbbacac79,36553,1732265730218-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:30,306 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:30,306 INFO [RS:2;296fbbacac79:36553 {}] regionserver.Replication(171): 296fbbacac79,36553,1732265730218 started 2024-11-22T08:55:30,308 DEBUG [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T08:55:30,308 INFO [RS_OPEN_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T08:55:30,308 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=296fbbacac79,34493,1732265727234 2024-11-22T08:55:30,309 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 296fbbacac79,34493,1732265727234, state=OPEN 2024-11-22T08:55:30,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36553-0x10066a4b12e0003, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:30,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:30,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T08:55:30,310 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=296fbbacac79,34493,1732265727234 2024-11-22T08:55:30,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:30,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:30,311 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T08:55:30,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-22T08:55:30,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=296fbbacac79,34493,1732265727234 in 210 msec 2024-11-22T08:55:30,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T08:55:30,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 369 msec 2024-11-22T08:55:30,315 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 296fbbacac79,41191,1732265727268 had 2 regions 2024-11-22T08:55:30,316 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 296fbbacac79,41191,1732265727268, splitWal=true, meta=true, isMeta: false 2024-11-22T08:55:30,316 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,41191,1732265727268-splitting dir is empty, no logs to split. 2024-11-22T08:55:30,317 INFO [PEWorker-3 {}] master.SplitWALManager(105): 296fbbacac79,41191,1732265727268 WAL count=0, meta=false 2024-11-22T08:55:30,318 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,41191,1732265727268-splitting dir is empty, no logs to split. 2024-11-22T08:55:30,318 INFO [PEWorker-3 {}] master.SplitWALManager(105): 296fbbacac79,41191,1732265727268 WAL count=0, meta=false 2024-11-22T08:55:30,318 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 296fbbacac79,41191,1732265727268 WAL splitting is done? wals=0, meta=false 2024-11-22T08:55:30,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732265728002, region=db7cb69e73f1bcaf57618c7b6060702d, ASSIGN}] 2024-11-22T08:55:30,320 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732265728002, region=db7cb69e73f1bcaf57618c7b6060702d, ASSIGN 2024-11-22T08:55:30,321 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732265728002, region=db7cb69e73f1bcaf57618c7b6060702d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T08:55:30,323 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T08:55:30,323 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(1482): Serving as 296fbbacac79,36553,1732265730218, RpcServer on 296fbbacac79/172.17.0.2:36553, sessionid=0x10066a4b12e0003 2024-11-22T08:55:30,323 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T08:55:30,324 DEBUG [RS:2;296fbbacac79:36553 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 296fbbacac79,36553,1732265730218 2024-11-22T08:55:30,324 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '296fbbacac79,36553,1732265730218' 2024-11-22T08:55:30,324 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T08:55:30,324 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T08:55:30,325 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T08:55:30,325 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T08:55:30,325 DEBUG [RS:2;296fbbacac79:36553 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 296fbbacac79,36553,1732265730218 2024-11-22T08:55:30,325 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '296fbbacac79,36553,1732265730218' 2024-11-22T08:55:30,325 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T08:55:30,325 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T08:55:30,326 DEBUG [RS:2;296fbbacac79:36553 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T08:55:30,326 INFO [RS:2;296fbbacac79:36553 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T08:55:30,326 INFO [RS:2;296fbbacac79:36553 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T08:55:30,426 INFO [RS:2;296fbbacac79:36553 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T08:55:30,427 INFO [RS:2;296fbbacac79:36553 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=296fbbacac79%2C36553%2C1732265730218, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,36553,1732265730218, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs, maxLogs=55 2024-11-22T08:55:30,444 INFO [RS:2;296fbbacac79:36553 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/WALs/296fbbacac79,36553,1732265730218/296fbbacac79%2C36553%2C1732265730218.1732265730428 2024-11-22T08:55:30,445 DEBUG [RS:2;296fbbacac79:36553 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T08:55:30,472 DEBUG [296fbbacac79:39331 {}] balancer.BalancerClusterState(204): Hosts are {296fbbacac79=0} racks are {/default-rack=0} 2024-11-22T08:55:30,472 DEBUG [296fbbacac79:39331 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T08:55:30,472 DEBUG [296fbbacac79:39331 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T08:55:30,472 DEBUG [296fbbacac79:39331 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T08:55:30,472 DEBUG [296fbbacac79:39331 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T08:55:30,472 INFO [296fbbacac79:39331 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T08:55:30,472 INFO [296fbbacac79:39331 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T08:55:30,472 DEBUG [296fbbacac79:39331 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T08:55:30,472 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=db7cb69e73f1bcaf57618c7b6060702d, regionState=OPENING, regionLocation=296fbbacac79,36553,1732265730218 2024-11-22T08:55:30,473 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 296fbbacac79:41191 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 296fbbacac79/172.17.0.2:41191 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T08:55:30,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=296fbbacac79,41191,1732265727268, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=296fbbacac79,41191,1732265727268, seqNum=-1, error=java.net.ConnectException: Call to address=296fbbacac79:41191 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 296fbbacac79/172.17.0.2:41191 2024-11-22T08:55:30,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=296fbbacac79,41191,1732265727268, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-22T08:55:30,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=296fbbacac79,41191,1732265727268, seqNum=-1 from cache 2024-11-22T08:55:30,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 296fbbacac79:41191 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 296fbbacac79/172.17.0.2:41191 2024-11-22T08:55:30,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T08:55:30,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=296fbbacac79,34493,1732265727234, seqNum=-1] 2024-11-22T08:55:30,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T08:55:30,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44523, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T08:55:30,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732265728002, region=db7cb69e73f1bcaf57618c7b6060702d, ASSIGN because future has completed 2024-11-22T08:55:30,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure db7cb69e73f1bcaf57618c7b6060702d, server=296fbbacac79,36553,1732265730218}] 2024-11-22T08:55:30,745 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T08:55:30,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50395, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T08:55:30,752 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:30,752 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => db7cb69e73f1bcaf57618c7b6060702d, NAME => 'testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d.', STARTKEY => '', ENDKEY => ''} 2024-11-22T08:55:30,753 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732265728002 db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,754 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T08:55:30,754 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,754 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,755 INFO [StoreOpener-db7cb69e73f1bcaf57618c7b6060702d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,756 INFO [StoreOpener-db7cb69e73f1bcaf57618c7b6060702d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db7cb69e73f1bcaf57618c7b6060702d columnFamilyName family 2024-11-22T08:55:30,756 DEBUG [StoreOpener-db7cb69e73f1bcaf57618c7b6060702d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T08:55:30,758 DEBUG [StoreFileOpener-db7cb69e73f1bcaf57618c7b6060702d-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7, delay=953 ms 2024-11-22T08:55:30,759 DEBUG [StoreOpener-db7cb69e73f1bcaf57618c7b6060702d-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7 2024-11-22T08:55:30,759 INFO [StoreOpener-db7cb69e73f1bcaf57618c7b6060702d-1 {}] regionserver.HStore(327): Store=db7cb69e73f1bcaf57618c7b6060702d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T08:55:30,760 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,761 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,761 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,761 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,761 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,763 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,763 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened db7cb69e73f1bcaf57618c7b6060702d; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71901745, jitterRate=0.07141949236392975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T08:55:30,763 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,764 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for db7cb69e73f1bcaf57618c7b6060702d: Running coprocessor pre-open hook at 1732265730754Writing region info on filesystem at 1732265730754Initializing all the Stores at 1732265730755 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732265730755Cleaning up temporary data from old regions at 1732265730762 (+7 ms)Running coprocessor post-open hooks at 1732265730763 (+1 ms)Region opened successfully at 1732265730764 (+1 ms) 2024-11-22T08:55:30,765 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d., pid=11, masterSystemTime=1732265730744 2024-11-22T08:55:30,768 DEBUG [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:30,768 INFO [RS_OPEN_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:30,768 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=db7cb69e73f1bcaf57618c7b6060702d, regionState=OPEN, openSeqNum=10, regionLocation=296fbbacac79,36553,1732265730218 2024-11-22T08:55:30,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure db7cb69e73f1bcaf57618c7b6060702d, server=296fbbacac79,36553,1732265730218 because future has completed 2024-11-22T08:55:30,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-22T08:55:30,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure db7cb69e73f1bcaf57618c7b6060702d, server=296fbbacac79,36553,1732265730218 in 181 msec 2024-11-22T08:55:30,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-22T08:55:30,776 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 296fbbacac79,41191,1732265727268 after splitting done 2024-11-22T08:55:30,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732265728002, region=db7cb69e73f1bcaf57618c7b6060702d, ASSIGN in 455 msec 2024-11-22T08:55:30,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 296fbbacac79,41191,1732265727268, splitWal=true, meta=true in 842 msec 2024-11-22T08:55:30,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T08:55:30,824 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T08:55:30,824 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T08:55:30,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:30,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:30,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T08:55:30,824 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T08:55:30,824 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=14869203, stopped=false 2024-11-22T08:55:30,825 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=296fbbacac79,39331,1732265727182 2024-11-22T08:55:30,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:30,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:30,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36553-0x10066a4b12e0003, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T08:55:30,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:30,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36553-0x10066a4b12e0003, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:30,826 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T08:55:30,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:30,826 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T08:55:30,826 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T08:55:30,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:30,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:30,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36553-0x10066a4b12e0003, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:30,827 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '296fbbacac79,34493,1732265727234' ***** 2024-11-22T08:55:30,827 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T08:55:30,827 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '296fbbacac79,36553,1732265730218' ***** 2024-11-22T08:55:30,827 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T08:55:30,827 INFO [RS:0;296fbbacac79:34493 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T08:55:30,827 INFO [RS:2;296fbbacac79:36553 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T08:55:30,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T08:55:30,827 INFO [RS:0;296fbbacac79:34493 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.persistence 2024-11-22T08:55:30,827 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T08:55:30,827 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T08:55:30,827 INFO [RS:2;296fbbacac79:36553 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a2db1b4-a454-1fff-a628-d625a88f0192/bucket.persistence 2024-11-22T08:55:30,827 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T08:55:30,827 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T08:55:30,827 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T08:55:30,827 INFO [RS:0;296fbbacac79:34493 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T08:55:30,827 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T08:55:30,828 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T08:55:30,828 INFO [RS:2;296fbbacac79:36553 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T08:55:30,828 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T08:55:30,838 DEBUG [RS:0;296fbbacac79:34493 {}] bucket.BucketCache(1487): Thread RS:0;296fbbacac79:34493 started persisting bucket cache to file 2024-11-22T08:55:30,838 DEBUG [RS:2;296fbbacac79:36553 {}] bucket.BucketCache(1487): Thread RS:2;296fbbacac79:36553 started persisting bucket cache to file 2024-11-22T08:55:30,838 DEBUG [RS:2;296fbbacac79:36553 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T08:55:30,838 DEBUG [RS:0;296fbbacac79:34493 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T08:55:30,838 DEBUG [RS:2;296fbbacac79:36553 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T08:55:30,838 DEBUG [RS:0;296fbbacac79:34493 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T08:55:30,842 DEBUG [RS:2;296fbbacac79:36553 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T08:55:30,842 DEBUG [RS:0;296fbbacac79:34493 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T08:55:30,842 DEBUG [RS:0;296fbbacac79:34493 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732265730838 2024-11-22T08:55:30,842 DEBUG [RS:2;296fbbacac79:36553 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732265730838 2024-11-22T08:55:30,842 DEBUG [RS:2;296fbbacac79:36553 {}] bucket.BucketCache(1509): Thread RS:2;296fbbacac79:36553 finished persisting bucket cache to file, renaming 2024-11-22T08:55:30,842 DEBUG [RS:0;296fbbacac79:34493 {}] bucket.BucketCache(1509): Thread RS:0;296fbbacac79:34493 finished persisting bucket cache to file, renaming 2024-11-22T08:55:30,842 WARN [RS:0;296fbbacac79:34493 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-22T08:55:30,842 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T08:55:30,842 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T08:55:30,842 INFO [RS:0;296fbbacac79:34493 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T08:55:30,842 INFO [RS:2;296fbbacac79:36553 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T08:55:30,842 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T08:55:30,842 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T08:55:30,842 INFO [RS:0;296fbbacac79:34493 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T08:55:30,842 INFO [RS:2;296fbbacac79:36553 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T08:55:30,842 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(959): stopping server 296fbbacac79,34493,1732265727234 2024-11-22T08:55:30,842 INFO [RS:0;296fbbacac79:34493 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T08:55:30,842 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(3091): Received CLOSE for db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,842 INFO [RS:0;296fbbacac79:34493 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;296fbbacac79:34493. 2024-11-22T08:55:30,843 DEBUG [RS:0;296fbbacac79:34493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T08:55:30,843 DEBUG [RS:0;296fbbacac79:34493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:30,843 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(959): stopping server 296fbbacac79,36553,1732265730218 2024-11-22T08:55:30,843 INFO [RS:0;296fbbacac79:34493 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T08:55:30,843 INFO [RS:0;296fbbacac79:34493 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T08:55:30,843 INFO [RS:2;296fbbacac79:36553 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T08:55:30,843 INFO [RS:0;296fbbacac79:34493 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T08:55:30,843 INFO [RS:2;296fbbacac79:36553 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;296fbbacac79:36553. 2024-11-22T08:55:30,843 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T08:55:30,843 DEBUG [RS:2;296fbbacac79:36553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T08:55:30,843 DEBUG [RS:2;296fbbacac79:36553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:30,843 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T08:55:30,843 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T08:55:30,843 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T08:55:30,843 DEBUG [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T08:55:30,843 DEBUG [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(1325): Online Regions={db7cb69e73f1bcaf57618c7b6060702d=testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d.} 2024-11-22T08:55:30,843 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing db7cb69e73f1bcaf57618c7b6060702d, disabling compactions & flushes 2024-11-22T08:55:30,843 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T08:55:30,843 DEBUG [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T08:55:30,843 DEBUG [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(1351): Waiting on db7cb69e73f1bcaf57618c7b6060702d 2024-11-22T08:55:30,843 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:30,843 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T08:55:30,843 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T08:55:30,843 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:30,843 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T08:55:30,843 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. after waiting 0 ms 2024-11-22T08:55:30,843 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T08:55:30,844 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T08:55:30,844 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:30,844 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-22T08:55:30,845 DEBUG [StoreFileCloser-db7cb69e73f1bcaf57618c7b6060702d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/family/276d2fff1875424cafc3768aa6dee2b7 2024-11-22T08:55:30,849 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/default/testBlockEvictionOnGracefulStop1732265728002/db7cb69e73f1bcaf57618c7b6060702d/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-22T08:55:30,850 INFO [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:30,850 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for db7cb69e73f1bcaf57618c7b6060702d: Waiting for close lock at 1732265730843Running coprocessor pre-close hooks at 1732265730843Disabling compacts and flushes for region at 1732265730843Disabling writes for close at 1732265730843Writing region close event to WAL at 1732265730845 (+2 ms)Running coprocessor post-close hooks at 1732265730850 (+5 ms)Closed at 1732265730850 2024-11-22T08:55:30,850 INFO [regionserver/296fbbacac79:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T08:55:30,850 DEBUG [RS_CLOSE_REGION-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d. 2024-11-22T08:55:30,853 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d30239c53f034827b422e077b4ac5293 2024-11-22T08:55:30,865 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/info/d30239c53f034827b422e077b4ac5293 is 209, key is testBlockEvictionOnGracefulStop1732265728002,,1732265728006.db7cb69e73f1bcaf57618c7b6060702d./info:regioninfo/1732265730768/Put/seqid=0 2024-11-22T08:55:30,866 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/info/d30239c53f034827b422e077b4ac5293 2024-11-22T08:55:30,868 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/.tmp/info/d30239c53f034827b422e077b4ac5293 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/d30239c53f034827b422e077b4ac5293 2024-11-22T08:55:30,869 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/d30239c53f034827b422e077b4ac5293, delay=964 ms 2024-11-22T08:55:30,870 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/d30239c53f034827b422e077b4ac5293, entries=8, sequenceid=19, filesize=6.6 K 2024-11-22T08:55:30,871 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 27ms, sequenceid=19, compaction requested=false 2024-11-22T08:55:30,872 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/262f5888ee854004a70697ea0a90d66c 2024-11-22T08:55:30,872 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/info/d30239c53f034827b422e077b4ac5293 2024-11-22T08:55:30,872 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/ns/2c16a3026c6f4944957680bd2bbd487c 2024-11-22T08:55:30,873 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/table/b23bf70213a64c9e92a106b0d1c6e784 2024-11-22T08:55:30,877 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-22T08:55:30,877 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T08:55:30,877 INFO [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T08:55:30,878 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732265730843Running coprocessor pre-close hooks at 1732265730843Disabling compacts and flushes for region at 1732265730843Disabling writes for close at 1732265730843Obtaining lock to block concurrent updates at 1732265730844 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732265730844Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732265730844Flushing stores of hbase:meta,,1.1588230740 at 1732265730844Flushing 1588230740/info: creating writer at 1732265730845 (+1 ms)Flushing 1588230740/info: appending metadata at 1732265730864 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732265730865 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4315ca45: reopening flushed file at 1732265730868 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 27ms, sequenceid=19, compaction requested=false at 1732265730871 (+3 ms)Writing region close event to WAL at 1732265730873 (+2 ms)Running coprocessor post-close hooks at 1732265730877 (+4 ms)Closed at 1732265730877 2024-11-22T08:55:30,878 DEBUG [RS_CLOSE_META-regionserver/296fbbacac79:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T08:55:30,891 INFO [regionserver/296fbbacac79:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T08:55:31,043 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(976): stopping server 296fbbacac79,36553,1732265730218; all regions closed. 2024-11-22T08:55:31,043 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(976): stopping server 296fbbacac79,34493,1732265727234; all regions closed. 2024-11-22T08:55:31,047 DEBUG [RS:2;296fbbacac79:36553 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs 2024-11-22T08:55:31,047 INFO [RS:2;296fbbacac79:36553 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 296fbbacac79%2C36553%2C1732265730218:(num 1732265730428) 2024-11-22T08:55:31,047 DEBUG [RS:2;296fbbacac79:36553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:31,047 DEBUG [RS:0;296fbbacac79:34493 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs 2024-11-22T08:55:31,048 INFO [RS:2;296fbbacac79:36553 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T08:55:31,048 INFO [RS:0;296fbbacac79:34493 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 296fbbacac79%2C34493%2C1732265727234.meta:.meta(num 1732265730258) 2024-11-22T08:55:31,048 INFO [RS:2;296fbbacac79:36553 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T08:55:31,048 INFO [RS:2;296fbbacac79:36553 {}] hbase.ChoreService(370): Chore service for: regionserver/296fbbacac79:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T08:55:31,049 INFO [RS:2;296fbbacac79:36553 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T08:55:31,049 INFO [regionserver/296fbbacac79:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T08:55:31,049 INFO [RS:2;296fbbacac79:36553 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T08:55:31,049 INFO [RS:2;296fbbacac79:36553 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T08:55:31,049 INFO [RS:2;296fbbacac79:36553 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T08:55:31,050 INFO [RS:2;296fbbacac79:36553 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36553 2024-11-22T08:55:31,051 DEBUG [RS:0;296fbbacac79:34493 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/oldWALs 2024-11-22T08:55:31,051 INFO [RS:0;296fbbacac79:34493 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 296fbbacac79%2C34493%2C1732265727234:(num 1732265727697) 2024-11-22T08:55:31,051 DEBUG [RS:0;296fbbacac79:34493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T08:55:31,052 INFO [RS:0;296fbbacac79:34493 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T08:55:31,052 INFO [RS:0;296fbbacac79:34493 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T08:55:31,052 INFO [RS:0;296fbbacac79:34493 {}] hbase.ChoreService(370): Chore service for: regionserver/296fbbacac79:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T08:55:31,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36553-0x10066a4b12e0003, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/296fbbacac79,36553,1732265730218 2024-11-22T08:55:31,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T08:55:31,052 INFO [RS:0;296fbbacac79:34493 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T08:55:31,052 INFO [RS:2;296fbbacac79:36553 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T08:55:31,052 INFO [regionserver/296fbbacac79:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T08:55:31,053 INFO [RS:0;296fbbacac79:34493 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34493 2024-11-22T08:55:31,054 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [296fbbacac79,36553,1732265730218] 2024-11-22T08:55:31,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T08:55:31,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/296fbbacac79,34493,1732265727234 2024-11-22T08:55:31,054 INFO [RS:0;296fbbacac79:34493 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T08:55:31,054 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/296fbbacac79,36553,1732265730218 already deleted, retry=false 2024-11-22T08:55:31,055 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 296fbbacac79,36553,1732265730218 expired; onlineServers=1 2024-11-22T08:55:31,055 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [296fbbacac79,34493,1732265727234] 2024-11-22T08:55:31,056 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/296fbbacac79,34493,1732265727234 already deleted, retry=false 2024-11-22T08:55:31,056 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 296fbbacac79,34493,1732265727234 expired; onlineServers=0 2024-11-22T08:55:31,056 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '296fbbacac79,39331,1732265727182' ***** 2024-11-22T08:55:31,056 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T08:55:31,056 INFO [M:0;296fbbacac79:39331 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T08:55:31,056 INFO [M:0;296fbbacac79:39331 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T08:55:31,056 DEBUG [M:0;296fbbacac79:39331 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T08:55:31,056 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T08:55:31,056 DEBUG [M:0;296fbbacac79:39331 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T08:55:31,056 DEBUG [master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.large.0-1732265727473 {}] cleaner.HFileCleaner(306): Exit Thread[master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.large.0-1732265727473,5,FailOnTimeoutGroup] 2024-11-22T08:55:31,056 DEBUG [master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.small.0-1732265727474 {}] cleaner.HFileCleaner(306): Exit Thread[master/296fbbacac79:0:becomeActiveMaster-HFileCleaner.small.0-1732265727474,5,FailOnTimeoutGroup] 2024-11-22T08:55:31,056 INFO [M:0;296fbbacac79:39331 {}] hbase.ChoreService(370): Chore service for: master/296fbbacac79:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T08:55:31,056 INFO [M:0;296fbbacac79:39331 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T08:55:31,056 DEBUG [M:0;296fbbacac79:39331 {}] master.HMaster(1795): Stopping service threads 2024-11-22T08:55:31,057 INFO [M:0;296fbbacac79:39331 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T08:55:31,057 INFO [M:0;296fbbacac79:39331 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T08:55:31,057 INFO [M:0;296fbbacac79:39331 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T08:55:31,057 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T08:55:31,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T08:55:31,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T08:55:31,057 DEBUG [M:0;296fbbacac79:39331 {}] zookeeper.ZKUtil(347): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T08:55:31,057 WARN [M:0;296fbbacac79:39331 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T08:55:31,058 INFO [M:0;296fbbacac79:39331 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/.lastflushedseqids 2024-11-22T08:55:31,061 INFO [M:0;296fbbacac79:39331 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T08:55:31,061 INFO [M:0;296fbbacac79:39331 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T08:55:31,061 DEBUG [M:0;296fbbacac79:39331 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T08:55:31,061 INFO [M:0;296fbbacac79:39331 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:31,061 DEBUG [M:0;296fbbacac79:39331 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:31,061 DEBUG [M:0;296fbbacac79:39331 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T08:55:31,061 DEBUG [M:0;296fbbacac79:39331 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:31,061 INFO [M:0;296fbbacac79:39331 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-22T08:55:31,070 DEBUG [M:0;296fbbacac79:39331 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b3d5d6353cc64c4ba61d97f2b30aeb8c 2024-11-22T08:55:31,082 DEBUG [M:0;296fbbacac79:39331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b3d5d6353cc64c4ba61d97f2b30aeb8c is 82, key is hbase:meta,,1/info:regioninfo/1732265730308/Put/seqid=0 2024-11-22T08:55:31,083 INFO [M:0;296fbbacac79:39331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b3d5d6353cc64c4ba61d97f2b30aeb8c 2024-11-22T08:55:31,089 DEBUG [M:0;296fbbacac79:39331 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 869272704b8f4b2d84dd2bf1c098b54a 2024-11-22T08:55:31,102 DEBUG [M:0;296fbbacac79:39331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/869272704b8f4b2d84dd2bf1c098b54a is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732265728386/Put/seqid=0 2024-11-22T08:55:31,102 INFO [M:0;296fbbacac79:39331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/869272704b8f4b2d84dd2bf1c098b54a 2024-11-22T08:55:31,108 DEBUG [M:0;296fbbacac79:39331 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 141b0fb7f5c34e4981beec97d6b2405e 2024-11-22T08:55:31,120 DEBUG [M:0;296fbbacac79:39331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/141b0fb7f5c34e4981beec97d6b2405e is 69, key is 296fbbacac79,34493,1732265727234/rs:state/1732265727523/Put/seqid=0 2024-11-22T08:55:31,121 INFO [M:0;296fbbacac79:39331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/141b0fb7f5c34e4981beec97d6b2405e 2024-11-22T08:55:31,122 INFO [M:0;296fbbacac79:39331 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 141b0fb7f5c34e4981beec97d6b2405e 2024-11-22T08:55:31,123 DEBUG [M:0;296fbbacac79:39331 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b3d5d6353cc64c4ba61d97f2b30aeb8c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b3d5d6353cc64c4ba61d97f2b30aeb8c 2024-11-22T08:55:31,124 INFO [M:0;296fbbacac79:39331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b3d5d6353cc64c4ba61d97f2b30aeb8c, entries=14, sequenceid=102, filesize=5.9 K 2024-11-22T08:55:31,124 DEBUG [M:0;296fbbacac79:39331 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/869272704b8f4b2d84dd2bf1c098b54a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/869272704b8f4b2d84dd2bf1c098b54a 2024-11-22T08:55:31,126 INFO [M:0;296fbbacac79:39331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/869272704b8f4b2d84dd2bf1c098b54a, entries=11, sequenceid=102, filesize=6.5 K 2024-11-22T08:55:31,126 DEBUG [M:0;296fbbacac79:39331 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/141b0fb7f5c34e4981beec97d6b2405e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/141b0fb7f5c34e4981beec97d6b2405e 2024-11-22T08:55:31,127 INFO [M:0;296fbbacac79:39331 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 141b0fb7f5c34e4981beec97d6b2405e 2024-11-22T08:55:31,127 INFO [M:0;296fbbacac79:39331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca838a34-317a-3237-ea80-87e175a93295/29065bc2-20e5-2408-8280-160c12c9de3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/141b0fb7f5c34e4981beec97d6b2405e, entries=3, sequenceid=102, filesize=5.3 K 2024-11-22T08:55:31,128 INFO [M:0;296fbbacac79:39331 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=102, compaction requested=false 2024-11-22T08:55:31,129 INFO [M:0;296fbbacac79:39331 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T08:55:31,130 DEBUG [M:0;296fbbacac79:39331 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732265731061Disabling compacts and flushes for region at 1732265731061Disabling writes for close at 1732265731061Obtaining lock to block concurrent updates at 1732265731061Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732265731061Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732265731062 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732265731062Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732265731062Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732265731082 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732265731082Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732265731084 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732265731101 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732265731101Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732265731103 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732265731120 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732265731120Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31dcd593: reopening flushed file at 1732265731123 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b9e1051: reopening flushed file at 1732265731124 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1590ecf: reopening flushed file at 1732265731126 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=102, compaction requested=false at 1732265731128 (+2 ms)Writing region close event to WAL at 1732265731129 (+1 ms)Closed at 1732265731129 2024-11-22T08:55:31,130 INFO [M:0;296fbbacac79:39331 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T08:55:31,130 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T08:55:31,131 INFO [M:0;296fbbacac79:39331 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39331 2024-11-22T08:55:31,131 INFO [M:0;296fbbacac79:39331 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T08:55:31,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36553-0x10066a4b12e0003, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:31,154 INFO [RS:2;296fbbacac79:36553 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T08:55:31,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36553-0x10066a4b12e0003, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:31,154 INFO [RS:2;296fbbacac79:36553 {}] regionserver.HRegionServer(1031): Exiting; stopping=296fbbacac79,36553,1732265730218; zookeeper connection closed. 2024-11-22T08:55:31,154 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@49a74fcf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@49a74fcf 2024-11-22T08:55:31,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:31,155 INFO [RS:0;296fbbacac79:34493 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T08:55:31,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34493-0x10066a4b12e0001, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:31,155 INFO [RS:0;296fbbacac79:34493 {}] regionserver.HRegionServer(1031): Exiting; stopping=296fbbacac79,34493,1732265727234; zookeeper connection closed. 2024-11-22T08:55:31,155 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@148a01 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@148a01 2024-11-22T08:55:31,155 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-22T08:55:31,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:31,232 INFO [M:0;296fbbacac79:39331 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T08:55:31,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39331-0x10066a4b12e0000, quorum=127.0.0.1:53512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T08:55:31,235 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T08:55:31,236 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T08:55:31,236 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T08:55:31,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=265 (was 288), ProcessCount=11 (was 11), AvailableMemoryMB=5089 (was 5174)