2024-11-20 19:48:13,168 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 19:48:13,179 main DEBUG Took 0.009235 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 19:48:13,179 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 19:48:13,180 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 19:48:13,187 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 19:48:13,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,198 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 19:48:13,214 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,215 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,216 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,216 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,217 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,218 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,219 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,220 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,220 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,221 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,222 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,222 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,222 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,223 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,223 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,223 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,224 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,224 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,224 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 19:48:13,225 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,225 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 19:48:13,227 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 19:48:13,228 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 19:48:13,230 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 19:48:13,230 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 19:48:13,231 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 19:48:13,232 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 19:48:13,240 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 19:48:13,243 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 19:48:13,244 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 19:48:13,245 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 19:48:13,245 main DEBUG createAppenders(={Console}) 2024-11-20 19:48:13,246 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-20 19:48:13,246 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 19:48:13,246 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-20 19:48:13,247 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 19:48:13,247 main DEBUG OutputStream closed 2024-11-20 19:48:13,247 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 19:48:13,247 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 19:48:13,248 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-20 19:48:13,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 19:48:13,327 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 19:48:13,328 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 19:48:13,329 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 19:48:13,329 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 19:48:13,330 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 19:48:13,330 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 19:48:13,331 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 19:48:13,331 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 19:48:13,331 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 19:48:13,332 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 19:48:13,332 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 19:48:13,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 19:48:13,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 19:48:13,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 19:48:13,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 19:48:13,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 19:48:13,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 19:48:13,338 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 19:48:13,338 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-20 19:48:13,338 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 19:48:13,339 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-20T19:48:13,353 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-20 19:48:13,357 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 19:48:13,357 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T19:48:13,611 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b 2024-11-20T19:48:13,636 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=5708 2024-11-20T19:48:13,749 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T19:48:13,866 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:13,886 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/cluster_0f780d9c-b588-9692-bfc4-dbcf15a5f3a5, deleteOnExit=true 2024-11-20T19:48:13,995 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/cluster_0f780d9c-b588-9692-bfc4-dbcf15a5f3a5/zookeeper_0, clientPort=50782, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/cluster_0f780d9c-b588-9692-bfc4-dbcf15a5f3a5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/cluster_0f780d9c-b588-9692-bfc4-dbcf15a5f3a5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T19:48:14,007 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50782 2024-11-20T19:48:14,007 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:14,093 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 with version=8 2024-11-20T19:48:14,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/hbase-staging 2024-11-20T19:48:14,199 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T19:48:14,493 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7200dd4cf518:0 server-side Connection retries=45 2024-11-20T19:48:14,504 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:14,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:14,514 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T19:48:14,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:14,515 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T19:48:14,656 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T19:48:14,747 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T19:48:14,784 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T19:48:14,795 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T19:48:14,799 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T19:48:14,842 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21134 (auto-detected) 2024-11-20T19:48:14,843 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T19:48:14,865 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41687 2024-11-20T19:48:14,889 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41687 connecting to ZooKeeper ensemble=127.0.0.1:50782 2024-11-20T19:48:14,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416870x0, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T19:48:14,929 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41687-0x10136ec2a990000 connected 2024-11-20T19:48:14,961 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:14,962 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:14,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:14,975 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927, hbase.cluster.distributed=false 2024-11-20T19:48:15,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T19:48:15,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41687 2024-11-20T19:48:15,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41687 2024-11-20T19:48:15,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41687 2024-11-20T19:48:15,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41687 2024-11-20T19:48:15,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41687 2024-11-20T19:48:15,114 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7200dd4cf518:0 server-side Connection retries=45 2024-11-20T19:48:15,117 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:15,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:15,119 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T19:48:15,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:15,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T19:48:15,123 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T19:48:15,125 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T19:48:15,126 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41785 2024-11-20T19:48:15,128 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41785 connecting to ZooKeeper ensemble=127.0.0.1:50782 2024-11-20T19:48:15,129 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:15,130 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:15,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417850x0, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T19:48:15,135 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:417850x0, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:15,136 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41785-0x10136ec2a990001 connected 2024-11-20T19:48:15,141 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T19:48:15,154 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/bucket.cache 2024-11-20T19:48:15,159 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T19:48:15,160 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T19:48:15,161 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T19:48:15,163 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T19:48:15,163 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T19:48:15,170 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T19:48:15,175 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T19:48:15,175 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/bucket.persistence is complete. 2024-11-20T19:48:15,178 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T19:48:15,180 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T19:48:15,187 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T19:48:15,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41785 2024-11-20T19:48:15,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41785 2024-11-20T19:48:15,197 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41785 2024-11-20T19:48:15,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41785 2024-11-20T19:48:15,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41785 2024-11-20T19:48:15,222 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7200dd4cf518:0 server-side Connection retries=45 2024-11-20T19:48:15,222 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:15,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:15,223 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T19:48:15,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:15,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T19:48:15,223 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T19:48:15,224 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T19:48:15,226 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36499 2024-11-20T19:48:15,228 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36499 connecting to ZooKeeper ensemble=127.0.0.1:50782 2024-11-20T19:48:15,229 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:15,230 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:15,235 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364990x0, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T19:48:15,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36499-0x10136ec2a990002 connected 2024-11-20T19:48:15,236 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:15,237 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T19:48:15,238 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/bucket.cache 2024-11-20T19:48:15,238 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T19:48:15,239 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T19:48:15,239 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T19:48:15,240 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T19:48:15,240 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T19:48:15,240 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T19:48:15,241 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T19:48:15,241 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/bucket.persistence is complete. 2024-11-20T19:48:15,241 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T19:48:15,243 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T19:48:15,245 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T19:48:15,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36499 2024-11-20T19:48:15,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36499 2024-11-20T19:48:15,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36499 2024-11-20T19:48:15,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36499 2024-11-20T19:48:15,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36499 2024-11-20T19:48:15,274 DEBUG [M:0;7200dd4cf518:41687 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7200dd4cf518:41687 2024-11-20T19:48:15,275 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7200dd4cf518,41687,1732132094282 2024-11-20T19:48:15,283 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:15,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:15,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:15,287 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7200dd4cf518,41687,1732132094282 2024-11-20T19:48:15,312 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T19:48:15,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:15,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T19:48:15,312 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:15,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:15,315 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T19:48:15,317 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7200dd4cf518,41687,1732132094282 from backup master directory 2024-11-20T19:48:15,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:15,320 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:15,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7200dd4cf518,41687,1732132094282 2024-11-20T19:48:15,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:15,322 WARN [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T19:48:15,323 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7200dd4cf518,41687,1732132094282 2024-11-20T19:48:15,325 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T19:48:15,329 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T19:48:15,339 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/hbase.id] with ID: 076b0141-9d2b-46af-97b7-9246204de0d3 2024-11-20T19:48:15,339 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/.tmp/hbase.id 2024-11-20T19:48:15,346 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/hbase.id] 2024-11-20T19:48:15,404 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:15,410 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T19:48:15,433 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-20T19:48:15,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:15,439 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:15,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:15,471 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T19:48:15,474 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T19:48:15,541 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T19:48:15,549 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T19:48:15,623 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store 2024-11-20T19:48:15,648 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T19:48:15,651 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:15,652 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T19:48:15,652 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:15,653 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:15,654 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T19:48:15,654 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:15,655 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:15,655 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732132095652Disabling compacts and flushes for region at 1732132095652Disabling writes for close at 1732132095654 (+2 ms)Writing region close event to WAL at 1732132095654Closed at 1732132095654 2024-11-20T19:48:15,656 WARN [master/7200dd4cf518:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/.initializing 2024-11-20T19:48:15,656 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/WALs/7200dd4cf518,41687,1732132094282 2024-11-20T19:48:15,671 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T19:48:15,684 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7200dd4cf518%2C41687%2C1732132094282, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/WALs/7200dd4cf518,41687,1732132094282, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/oldWALs, maxLogs=10 2024-11-20T19:48:15,725 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/WALs/7200dd4cf518,41687,1732132094282/7200dd4cf518%2C41687%2C1732132094282.1732132095690 2024-11-20T19:48:15,726 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T19:48:15,726 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T19:48:15,727 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:15,731 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,732 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,769 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,798 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T19:48:15,802 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:15,803 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:15,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T19:48:15,810 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:15,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:15,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T19:48:15,815 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:15,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:15,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T19:48:15,821 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:15,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:15,822 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,826 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,826 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,829 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,830 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,832 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T19:48:15,835 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:15,842 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T19:48:15,843 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61104333, jitterRate=-0.08947448432445526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T19:48:15,852 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732132095744Initializing all the Stores at 1732132095745 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132095745Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132095747 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132095747Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132095747Cleaning up temporary data from old regions at 1732132095830 (+83 ms)Region opened successfully at 1732132095852 (+22 ms) 2024-11-20T19:48:15,854 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T19:48:15,893 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3288fea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7200dd4cf518/172.17.0.2:0 2024-11-20T19:48:15,927 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T19:48:15,941 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T19:48:15,941 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T19:48:15,944 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T19:48:15,945 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T19:48:15,952 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T19:48:15,952 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T19:48:15,982 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T19:48:15,992 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T19:48:15,995 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T19:48:15,998 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T19:48:15,999 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T19:48:16,001 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T19:48:16,003 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T19:48:16,006 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T19:48:16,008 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T19:48:16,009 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T19:48:16,012 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T19:48:16,036 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T19:48:16,038 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T19:48:16,050 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:16,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:16,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:16,051 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,054 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7200dd4cf518,41687,1732132094282, sessionid=0x10136ec2a990000, setting cluster-up flag (Was=false) 2024-11-20T19:48:16,067 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,074 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T19:48:16,076 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7200dd4cf518,41687,1732132094282 2024-11-20T19:48:16,081 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,093 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T19:48:16,094 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7200dd4cf518,41687,1732132094282 2024-11-20T19:48:16,100 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T19:48:16,162 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(746): ClusterId : 076b0141-9d2b-46af-97b7-9246204de0d3 2024-11-20T19:48:16,162 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(746): ClusterId : 076b0141-9d2b-46af-97b7-9246204de0d3 2024-11-20T19:48:16,165 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T19:48:16,165 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T19:48:16,171 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T19:48:16,171 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T19:48:16,171 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T19:48:16,171 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T19:48:16,176 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T19:48:16,176 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T19:48:16,177 DEBUG [RS:1;7200dd4cf518:36499 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dbf33c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7200dd4cf518/172.17.0.2:0 2024-11-20T19:48:16,177 DEBUG [RS:0;7200dd4cf518:41785 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26da4856, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7200dd4cf518/172.17.0.2:0 2024-11-20T19:48:16,181 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T19:48:16,192 DEBUG [RS:1;7200dd4cf518:36499 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7200dd4cf518:36499 2024-11-20T19:48:16,195 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T19:48:16,196 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T19:48:16,196 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T19:48:16,196 DEBUG [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T19:48:16,198 DEBUG [RS:0;7200dd4cf518:41785 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7200dd4cf518:41785 2024-11-20T19:48:16,198 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T19:48:16,198 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T19:48:16,199 DEBUG [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T19:48:16,199 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(2659): reportForDuty to master=7200dd4cf518,41687,1732132094282 with port=36499, startcode=1732132095222 2024-11-20T19:48:16,200 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(2659): reportForDuty to master=7200dd4cf518,41687,1732132094282 with port=41785, startcode=1732132095067 2024-11-20T19:48:16,205 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T19:48:16,213 DEBUG [RS:1;7200dd4cf518:36499 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T19:48:16,213 DEBUG [RS:0;7200dd4cf518:41785 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T19:48:16,211 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7200dd4cf518,41687,1732132094282 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T19:48:16,219 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7200dd4cf518:0, corePoolSize=5, maxPoolSize=5 2024-11-20T19:48:16,219 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7200dd4cf518:0, corePoolSize=5, maxPoolSize=5 2024-11-20T19:48:16,219 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7200dd4cf518:0, corePoolSize=5, maxPoolSize=5 2024-11-20T19:48:16,220 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7200dd4cf518:0, corePoolSize=5, maxPoolSize=5 2024-11-20T19:48:16,220 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7200dd4cf518:0, corePoolSize=10, maxPoolSize=10 2024-11-20T19:48:16,221 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,221 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7200dd4cf518:0, corePoolSize=2, maxPoolSize=2 2024-11-20T19:48:16,221 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,224 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732132126224 2024-11-20T19:48:16,226 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T19:48:16,226 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T19:48:16,227 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T19:48:16,228 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T19:48:16,231 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T19:48:16,232 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T19:48:16,232 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:16,232 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T19:48:16,232 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T19:48:16,232 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T19:48:16,233 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,244 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T19:48:16,245 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T19:48:16,246 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T19:48:16,248 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T19:48:16,249 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T19:48:16,252 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.large.0-1732132096250,5,FailOnTimeoutGroup] 2024-11-20T19:48:16,256 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.small.0-1732132096252,5,FailOnTimeoutGroup] 2024-11-20T19:48:16,257 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,257 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T19:48:16,258 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,258 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,269 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T19:48:16,270 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 2024-11-20T19:48:16,286 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:16,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T19:48:16,292 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44473, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T19:48:16,292 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56845, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T19:48:16,293 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T19:48:16,293 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:16,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:16,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T19:48:16,298 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T19:48:16,298 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:16,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:16,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T19:48:16,301 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41687 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7200dd4cf518,36499,1732132095222 2024-11-20T19:48:16,303 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T19:48:16,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:16,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:16,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T19:48:16,305 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41687 {}] master.ServerManager(517): Registering regionserver=7200dd4cf518,36499,1732132095222 2024-11-20T19:48:16,309 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T19:48:16,309 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:16,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:16,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T19:48:16,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740 2024-11-20T19:48:16,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740 2024-11-20T19:48:16,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T19:48:16,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T19:48:16,312 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T19:48:16,313 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T19:48:16,318 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41687 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7200dd4cf518,41785,1732132095067 2024-11-20T19:48:16,319 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41687 {}] master.ServerManager(517): Registering regionserver=7200dd4cf518,41785,1732132095067 2024-11-20T19:48:16,321 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T19:48:16,321 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61020247, jitterRate=-0.09072746336460114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T19:48:16,323 DEBUG [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 2024-11-20T19:48:16,324 DEBUG [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T19:48:16,324 DEBUG [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T19:48:16,324 DEBUG [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 2024-11-20T19:48:16,324 DEBUG [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T19:48:16,325 DEBUG [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T19:48:16,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732132096286Initializing all the Stores at 1732132096286Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132096287 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132096287Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132096287Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132096287Cleaning up temporary data from old regions at 1732132096312 (+25 ms)Region opened successfully at 1732132096326 (+14 ms) 2024-11-20T19:48:16,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T19:48:16,326 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T19:48:16,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T19:48:16,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T19:48:16,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T19:48:16,328 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T19:48:16,328 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732132096326Disabling compacts and flushes for region at 1732132096326Disabling writes for close at 1732132096327 (+1 ms)Writing region close event to WAL at 1732132096328 (+1 ms)Closed at 1732132096328 2024-11-20T19:48:16,331 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T19:48:16,331 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T19:48:16,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T19:48:16,334 DEBUG [RS:1;7200dd4cf518:36499 {}] zookeeper.ZKUtil(111): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7200dd4cf518,36499,1732132095222 2024-11-20T19:48:16,335 WARN [RS:1;7200dd4cf518:36499 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T19:48:16,335 DEBUG [RS:0;7200dd4cf518:41785 {}] zookeeper.ZKUtil(111): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7200dd4cf518,41785,1732132095067 2024-11-20T19:48:16,335 WARN [RS:0;7200dd4cf518:41785 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T19:48:16,335 INFO [RS:1;7200dd4cf518:36499 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T19:48:16,335 INFO [RS:0;7200dd4cf518:41785 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T19:48:16,335 DEBUG [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36499,1732132095222 2024-11-20T19:48:16,335 DEBUG [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,41785,1732132095067 2024-11-20T19:48:16,339 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7200dd4cf518,36499,1732132095222] 2024-11-20T19:48:16,339 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7200dd4cf518,41785,1732132095067] 2024-11-20T19:48:16,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T19:48:16,349 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T19:48:16,351 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T19:48:16,364 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T19:48:16,364 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T19:48:16,378 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T19:48:16,378 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T19:48:16,384 INFO [RS:0;7200dd4cf518:41785 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T19:48:16,384 INFO [RS:1;7200dd4cf518:36499 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T19:48:16,384 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,384 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,385 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T19:48:16,385 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T19:48:16,390 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T19:48:16,390 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T19:48:16,391 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,391 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,392 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,392 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,392 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,392 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,392 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,392 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,392 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,392 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,393 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,393 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7200dd4cf518:0, corePoolSize=2, maxPoolSize=2 2024-11-20T19:48:16,393 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,393 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,393 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,393 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7200dd4cf518:0, corePoolSize=2, maxPoolSize=2 2024-11-20T19:48:16,393 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,393 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,393 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,393 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,394 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,394 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,394 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7200dd4cf518:0, corePoolSize=3, maxPoolSize=3 2024-11-20T19:48:16,394 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,394 DEBUG [RS:1;7200dd4cf518:36499 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7200dd4cf518:0, corePoolSize=3, maxPoolSize=3 2024-11-20T19:48:16,394 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,394 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,395 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:16,395 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7200dd4cf518:0, corePoolSize=3, maxPoolSize=3 2024-11-20T19:48:16,395 DEBUG [RS:0;7200dd4cf518:41785 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7200dd4cf518:0, corePoolSize=3, maxPoolSize=3 2024-11-20T19:48:16,396 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,396 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,396 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,396 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,396 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,396 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,397 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,397 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,397 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,36499,1732132095222-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T19:48:16,397 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,397 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,397 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,41785,1732132095067-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T19:48:16,418 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T19:48:16,418 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T19:48:16,420 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,36499,1732132095222-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,420 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,41785,1732132095067-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,421 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,421 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,421 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.Replication(171): 7200dd4cf518,36499,1732132095222 started 2024-11-20T19:48:16,421 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.Replication(171): 7200dd4cf518,41785,1732132095067 started 2024-11-20T19:48:16,442 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,442 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:16,443 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(1482): Serving as 7200dd4cf518,36499,1732132095222, RpcServer on 7200dd4cf518/172.17.0.2:36499, sessionid=0x10136ec2a990002 2024-11-20T19:48:16,443 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(1482): Serving as 7200dd4cf518,41785,1732132095067, RpcServer on 7200dd4cf518/172.17.0.2:41785, sessionid=0x10136ec2a990001 2024-11-20T19:48:16,444 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T19:48:16,444 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T19:48:16,444 DEBUG [RS:1;7200dd4cf518:36499 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7200dd4cf518,36499,1732132095222 2024-11-20T19:48:16,444 DEBUG [RS:0;7200dd4cf518:41785 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7200dd4cf518,41785,1732132095067 2024-11-20T19:48:16,444 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7200dd4cf518,36499,1732132095222' 2024-11-20T19:48:16,444 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7200dd4cf518,41785,1732132095067' 2024-11-20T19:48:16,444 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T19:48:16,444 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T19:48:16,445 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T19:48:16,445 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T19:48:16,446 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T19:48:16,446 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T19:48:16,446 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T19:48:16,446 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T19:48:16,447 DEBUG [RS:0;7200dd4cf518:41785 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7200dd4cf518,41785,1732132095067 2024-11-20T19:48:16,447 DEBUG [RS:1;7200dd4cf518:36499 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7200dd4cf518,36499,1732132095222 2024-11-20T19:48:16,447 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7200dd4cf518,41785,1732132095067' 2024-11-20T19:48:16,447 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7200dd4cf518,36499,1732132095222' 2024-11-20T19:48:16,447 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T19:48:16,447 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T19:48:16,447 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T19:48:16,447 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T19:48:16,448 DEBUG [RS:1;7200dd4cf518:36499 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T19:48:16,448 DEBUG [RS:0;7200dd4cf518:41785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T19:48:16,448 INFO [RS:1;7200dd4cf518:36499 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T19:48:16,448 INFO [RS:0;7200dd4cf518:41785 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T19:48:16,448 INFO [RS:1;7200dd4cf518:36499 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T19:48:16,448 INFO [RS:0;7200dd4cf518:41785 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T19:48:16,502 WARN [7200dd4cf518:41687 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T19:48:16,554 INFO [RS:1;7200dd4cf518:36499 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T19:48:16,554 INFO [RS:0;7200dd4cf518:41785 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T19:48:16,555 INFO [RS:1;7200dd4cf518:36499 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7200dd4cf518%2C36499%2C1732132095222, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36499,1732132095222, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs, maxLogs=55 2024-11-20T19:48:16,555 INFO [RS:0;7200dd4cf518:41785 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7200dd4cf518%2C41785%2C1732132095067, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,41785,1732132095067, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs, maxLogs=55 2024-11-20T19:48:16,579 INFO [RS:1;7200dd4cf518:36499 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36499,1732132095222/7200dd4cf518%2C36499%2C1732132095222.1732132096559 2024-11-20T19:48:16,579 DEBUG [RS:1;7200dd4cf518:36499 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T19:48:16,579 INFO [RS:0;7200dd4cf518:41785 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,41785,1732132095067/7200dd4cf518%2C41785%2C1732132095067.1732132096559 2024-11-20T19:48:16,580 DEBUG [RS:0;7200dd4cf518:41785 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T19:48:16,755 DEBUG [7200dd4cf518:41687 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T19:48:16,770 DEBUG [7200dd4cf518:41687 {}] balancer.BalancerClusterState(204): Hosts are {7200dd4cf518=0} racks are {/default-rack=0} 2024-11-20T19:48:16,775 DEBUG [7200dd4cf518:41687 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T19:48:16,775 DEBUG [7200dd4cf518:41687 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T19:48:16,775 DEBUG [7200dd4cf518:41687 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T19:48:16,775 DEBUG [7200dd4cf518:41687 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T19:48:16,775 INFO [7200dd4cf518:41687 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T19:48:16,775 INFO [7200dd4cf518:41687 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T19:48:16,775 DEBUG [7200dd4cf518:41687 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T19:48:16,783 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7200dd4cf518,36499,1732132095222 2024-11-20T19:48:16,789 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7200dd4cf518,36499,1732132095222, state=OPENING 2024-11-20T19:48:16,794 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T19:48:16,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,797 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:16,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:16,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:16,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:16,800 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T19:48:16,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7200dd4cf518,36499,1732132095222}] 2024-11-20T19:48:16,977 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T19:48:16,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37347, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T19:48:16,989 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T19:48:16,990 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T19:48:16,991 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T19:48:16,992 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7200dd4cf518%2C36499%2C1732132095222.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36499,1732132095222, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs, maxLogs=55 2024-11-20T19:48:17,014 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36499,1732132095222/7200dd4cf518%2C36499%2C1732132095222.meta.1732132096994.meta 2024-11-20T19:48:17,015 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T19:48:17,015 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T19:48:17,017 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T19:48:17,019 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T19:48:17,024 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T19:48:17,028 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T19:48:17,029 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:17,029 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T19:48:17,029 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T19:48:17,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T19:48:17,032 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T19:48:17,032 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:17,032 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:17,032 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T19:48:17,033 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T19:48:17,033 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:17,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:17,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T19:48:17,034 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T19:48:17,035 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:17,035 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:17,035 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T19:48:17,036 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T19:48:17,036 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:17,036 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:17,036 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T19:48:17,038 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740 2024-11-20T19:48:17,039 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740 2024-11-20T19:48:17,039 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T19:48:17,039 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T19:48:17,039 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T19:48:17,041 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T19:48:17,041 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74636226, jitterRate=0.11216643452644348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T19:48:17,041 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T19:48:17,043 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732132097030Writing region info on filesystem at 1732132097030Initializing all the Stores at 1732132097030Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132097030Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132097031 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132097031Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132097031Cleaning up temporary data from old regions at 1732132097039 (+8 ms)Running coprocessor post-open hooks at 1732132097042 (+3 ms)Region opened successfully at 1732132097043 (+1 ms) 2024-11-20T19:48:17,050 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732132096969 2024-11-20T19:48:17,062 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T19:48:17,063 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T19:48:17,064 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7200dd4cf518,36499,1732132095222 2024-11-20T19:48:17,065 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7200dd4cf518,36499,1732132095222, state=OPEN 2024-11-20T19:48:17,068 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:17,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:17,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:17,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:17,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:17,069 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7200dd4cf518,36499,1732132095222 2024-11-20T19:48:17,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:17,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T19:48:17,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7200dd4cf518,36499,1732132095222 in 267 msec 2024-11-20T19:48:17,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T19:48:17,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 735 msec 2024-11-20T19:48:17,078 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T19:48:17,078 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T19:48:17,099 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T19:48:17,100 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7200dd4cf518,36499,1732132095222, seqNum=-1] 2024-11-20T19:48:17,123 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T19:48:17,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36239, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T19:48:17,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0130 sec 2024-11-20T19:48:17,142 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732132097142, completionTime=-1 2024-11-20T19:48:17,144 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T19:48:17,145 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T19:48:17,172 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T19:48:17,172 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732132157172 2024-11-20T19:48:17,172 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732132217172 2024-11-20T19:48:17,172 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-20T19:48:17,174 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T19:48:17,181 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,41687,1732132094282-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:17,181 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,41687,1732132094282-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:17,181 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,41687,1732132094282-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:17,183 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7200dd4cf518:41687, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:17,184 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:17,184 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:17,189 DEBUG [master/7200dd4cf518:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T19:48:17,212 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.888sec 2024-11-20T19:48:17,213 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T19:48:17,215 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T19:48:17,216 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T19:48:17,216 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T19:48:17,216 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T19:48:17,217 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,41687,1732132094282-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T19:48:17,217 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,41687,1732132094282-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T19:48:17,219 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T19:48:17,220 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T19:48:17,220 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,41687,1732132094282-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:17,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8eba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T19:48:17,277 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T19:48:17,277 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T19:48:17,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7200dd4cf518,41687,-1 for getting cluster id 2024-11-20T19:48:17,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T19:48:17,320 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '076b0141-9d2b-46af-97b7-9246204de0d3' 2024-11-20T19:48:17,323 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T19:48:17,324 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "076b0141-9d2b-46af-97b7-9246204de0d3" 2024-11-20T19:48:17,326 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d7bad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T19:48:17,326 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7200dd4cf518,41687,-1] 2024-11-20T19:48:17,329 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T19:48:17,331 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:17,332 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T19:48:17,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T19:48:17,336 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T19:48:17,342 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7200dd4cf518,36499,1732132095222, seqNum=-1] 2024-11-20T19:48:17,343 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T19:48:17,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T19:48:17,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7200dd4cf518,41687,1732132094282 2024-11-20T19:48:17,368 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T19:48:17,372 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7200dd4cf518,41687,1732132094282 2024-11-20T19:48:17,375 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63611b64 2024-11-20T19:48:17,376 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T19:48:17,378 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T19:48:17,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732132097364', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T19:48:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732132097364 2024-11-20T19:48:17,395 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732132097364 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T19:48:17,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732132097364" procId is: 4 2024-11-20T19:48:17,398 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:17,399 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732132097364 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T19:48:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T19:48:17,414 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732132097364-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5bcb6f0185137adf4e79c74b1424d03d, NAME => 'testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732132097364', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 2024-11-20T19:48:17,422 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732132097364-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:17,422 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732132097364-pool-0 {}] regionserver.HRegion(1722): Closing 5bcb6f0185137adf4e79c74b1424d03d, disabling compactions & flushes 2024-11-20T19:48:17,422 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732132097364-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:17,422 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732132097364-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:17,422 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732132097364-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. after waiting 0 ms 2024-11-20T19:48:17,422 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732132097364-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:17,422 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732132097364-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:17,422 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732132097364-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5bcb6f0185137adf4e79c74b1424d03d: Waiting for close lock at 1732132097422Disabling compacts and flushes for region at 1732132097422Disabling writes for close at 1732132097422Writing region close event to WAL at 1732132097422Closed at 1732132097422 2024-11-20T19:48:17,424 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732132097364 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T19:48:17,429 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732132097424"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732132097424"}]},"ts":"1732132097424"} 2024-11-20T19:48:17,433 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T19:48:17,434 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732132097364 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T19:48:17,437 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732132097364","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732132097434"}]},"ts":"1732132097434"} 2024-11-20T19:48:17,441 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732132097364, state=ENABLING in hbase:meta 2024-11-20T19:48:17,441 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {7200dd4cf518=0} racks are {/default-rack=0} 2024-11-20T19:48:17,442 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T19:48:17,442 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T19:48:17,442 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T19:48:17,442 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T19:48:17,442 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T19:48:17,442 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T19:48:17,442 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T19:48:17,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, ASSIGN}] 2024-11-20T19:48:17,445 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, ASSIGN 2024-11-20T19:48:17,446 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, ASSIGN; state=OFFLINE, location=7200dd4cf518,41785,1732132095067; forceNewPlan=false, retain=false 2024-11-20T19:48:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T19:48:17,598 INFO [7200dd4cf518:41687 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T19:48:17,599 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5bcb6f0185137adf4e79c74b1424d03d, regionState=OPENING, regionLocation=7200dd4cf518,41785,1732132095067 2024-11-20T19:48:17,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, ASSIGN because future has completed 2024-11-20T19:48:17,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bcb6f0185137adf4e79c74b1424d03d, server=7200dd4cf518,41785,1732132095067}] 2024-11-20T19:48:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T19:48:17,757 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T19:48:17,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58047, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T19:48:17,761 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:17,762 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bcb6f0185137adf4e79c74b1424d03d, NAME => 'testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d.', STARTKEY => '', ENDKEY => ''} 2024-11-20T19:48:17,762 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732132097364 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,762 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:17,762 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,762 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,764 INFO [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,768 INFO [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bcb6f0185137adf4e79c74b1424d03d columnFamilyName family 2024-11-20T19:48:17,768 DEBUG [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:17,769 INFO [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] regionserver.HStore(327): Store=5bcb6f0185137adf4e79c74b1424d03d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:17,769 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,771 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,771 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,771 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,771 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,772 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,779 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T19:48:17,779 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5bcb6f0185137adf4e79c74b1424d03d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73844174, jitterRate=0.10036394000053406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T19:48:17,780 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:17,781 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5bcb6f0185137adf4e79c74b1424d03d: Running coprocessor pre-open hook at 1732132097763Writing region info on filesystem at 1732132097763Initializing all the Stores at 1732132097763Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132097763Cleaning up temporary data from old regions at 1732132097771 (+8 ms)Running coprocessor post-open hooks at 1732132097780 (+9 ms)Region opened successfully at 1732132097781 (+1 ms) 2024-11-20T19:48:17,782 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d., pid=6, masterSystemTime=1732132097756 2024-11-20T19:48:17,785 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:17,785 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:17,786 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5bcb6f0185137adf4e79c74b1424d03d, regionState=OPEN, openSeqNum=2, regionLocation=7200dd4cf518,41785,1732132095067 2024-11-20T19:48:17,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bcb6f0185137adf4e79c74b1424d03d, server=7200dd4cf518,41785,1732132095067 because future has completed 2024-11-20T19:48:17,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T19:48:17,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bcb6f0185137adf4e79c74b1424d03d, server=7200dd4cf518,41785,1732132095067 in 186 msec 2024-11-20T19:48:17,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T19:48:17,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, ASSIGN in 349 msec 2024-11-20T19:48:17,795 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732132097364 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T19:48:17,796 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732132097364","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732132097796"}]},"ts":"1732132097796"} 2024-11-20T19:48:17,798 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732132097364, state=ENABLED in hbase:meta 2024-11-20T19:48:17,799 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732132097364 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T19:48:17,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732132097364 in 411 msec 2024-11-20T19:48:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T19:48:18,031 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732132097364 get assigned. Timeout = 60000ms 2024-11-20T19:48:18,031 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732132097364 completed 2024-11-20T19:48:18,034 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T19:48:18,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732132097364 assigned to meta. Checking AM states. 2024-11-20T19:48:18,039 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T19:48:18,040 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732132097364 assigned. 2024-11-20T19:48:18,048 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732132097364', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d., hostname=7200dd4cf518,41785,1732132095067, seqNum=2] 2024-11-20T19:48:18,049 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T19:48:18,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T19:48:18,062 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5bcb6f0185137adf4e79c74b1424d03d 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T19:48:18,086 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e13cbf40635c45888c0fbe0a4b519bba 2024-11-20T19:48:18,128 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/.tmp/family/e13cbf40635c45888c0fbe0a4b519bba is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T19:48:18,134 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/.tmp/family/e13cbf40635c45888c0fbe0a4b519bba 2024-11-20T19:48:18,172 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/.tmp/family/e13cbf40635c45888c0fbe0a4b519bba as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba 2024-11-20T19:48:18,173 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba, delay=974 ms 2024-11-20T19:48:18,175 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T19:48:18,181 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 5bcb6f0185137adf4e79c74b1424d03d in 119ms, sequenceid=6, compaction requested=false 2024-11-20T19:48:18,182 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732132097364' 2024-11-20T19:48:18,184 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5bcb6f0185137adf4e79c74b1424d03d: 2024-11-20T19:48:19,150 DEBUG [hfile-prefetch-1732132098173 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key e13cbf40635c45888c0fbe0a4b519bba_0. 2024-11-20T19:48:19,154 DEBUG [hfile-prefetch-1732132098173 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key e13cbf40635c45888c0fbe0a4b519bba_117. 2024-11-20T19:48:19,154 DEBUG [hfile-prefetch-1732132098173 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba, with total blocks 2, and data blocks 1 2024-11-20T19:48:19,154 DEBUG [hfile-prefetch-1732132098173 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T19:48:19,155 DEBUG [hfile-prefetch-1732132098173 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T19:48:19,155 DEBUG [hfile-prefetch-1732132098173 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba. Will sleep 100ms and try the verification again. 2024-11-20T19:48:19,165 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T19:48:19,165 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T19:48:19,165 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T19:48:19,165 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T19:48:19,183 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-20T19:48:19,184 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732132099165 2024-11-20T19:48:19,184 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T19:48:19,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=5bcb6f0185137adf4e79c74b1424d03d, source=7200dd4cf518,41785,1732132095067, destination=7200dd4cf518,36499,1732132095222, warming up region on 7200dd4cf518,36499,1732132095222 2024-11-20T19:48:19,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=5bcb6f0185137adf4e79c74b1424d03d, source=7200dd4cf518,41785,1732132095067, destination=7200dd4cf518,36499,1732132095222, running balancer 2024-11-20T19:48:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, REOPEN/MOVE 2024-11-20T19:48:19,197 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, REOPEN/MOVE 2024-11-20T19:48:19,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36499 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36499 {}] regionserver.HRegion(7855): Warmup {ENCODED => 5bcb6f0185137adf4e79c74b1424d03d, NAME => 'testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d.', STARTKEY => '', ENDKEY => ''} 2024-11-20T19:48:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36499 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:19,198 INFO [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,199 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5bcb6f0185137adf4e79c74b1424d03d, regionState=CLOSING, regionLocation=7200dd4cf518,41785,1732132095067 2024-11-20T19:48:19,199 INFO [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bcb6f0185137adf4e79c74b1424d03d columnFamilyName family 2024-11-20T19:48:19,199 DEBUG [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:19,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, REOPEN/MOVE because future has completed 2024-11-20T19:48:19,202 DEBUG [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba 2024-11-20T19:48:19,203 INFO [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] regionserver.HStore(327): Store=5bcb6f0185137adf4e79c74b1424d03d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36499 {}] regionserver.HRegion(1722): Closing 5bcb6f0185137adf4e79c74b1424d03d, disabling compactions & flushes 2024-11-20T19:48:19,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36499 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36499 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36499 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. after waiting 0 ms 2024-11-20T19:48:19,203 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-20T19:48:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36499 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bcb6f0185137adf4e79c74b1424d03d, server=7200dd4cf518,41785,1732132095067}] 2024-11-20T19:48:19,204 DEBUG [StoreFileCloser-5bcb6f0185137adf4e79c74b1424d03d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba 2024-11-20T19:48:19,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36499 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36499 {}] regionserver.HRegion(1676): Region close journal for 5bcb6f0185137adf4e79c74b1424d03d: Waiting for close lock at 1732132099203Disabling compacts and flushes for region at 1732132099203Disabling writes for close at 1732132099203Writing region close event to WAL at 1732132099205 (+2 ms)Closed at 1732132099205 2024-11-20T19:48:19,207 WARN [hfile-prefetch-1732132098173 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-20T19:48:19,207 DEBUG [hfile-prefetch-1732132098173 {}] hfile.PrefetchExecutor(121): Prefetch completed for e13cbf40635c45888c0fbe0a4b519bba 2024-11-20T19:48:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41687 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T19:48:19,361 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,361 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-20T19:48:19,362 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 5bcb6f0185137adf4e79c74b1424d03d, disabling compactions & flushes 2024-11-20T19:48:19,362 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,362 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,362 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. after waiting 0 ms 2024-11-20T19:48:19,362 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,364 DEBUG [StoreFileCloser-5bcb6f0185137adf4e79c74b1424d03d-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file e13cbf40635c45888c0fbe0a4b519bba, starting offset: 0, end offset: 9223372036854775807 2024-11-20T19:48:19,365 DEBUG [StoreFileCloser-5bcb6f0185137adf4e79c74b1424d03d-family-1 {}] bucket.BucketCache(856): removed key e13cbf40635c45888c0fbe0a4b519bba_0 from back map with offset lock 2094080 in the evict process 2024-11-20T19:48:19,370 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T19:48:19,371 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,372 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 5bcb6f0185137adf4e79c74b1424d03d: Waiting for close lock at 1732132099362Running coprocessor pre-close hooks at 1732132099362Disabling compacts and flushes for region at 1732132099362Disabling writes for close at 1732132099362Writing region close event to WAL at 1732132099366 (+4 ms)Running coprocessor post-close hooks at 1732132099370 (+4 ms)Closed at 1732132099371 (+1 ms) 2024-11-20T19:48:19,372 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 5bcb6f0185137adf4e79c74b1424d03d move to 7200dd4cf518,36499,1732132095222 record at close sequenceid=6 2024-11-20T19:48:19,375 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,375 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5bcb6f0185137adf4e79c74b1424d03d, regionState=CLOSED 2024-11-20T19:48:19,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bcb6f0185137adf4e79c74b1424d03d, server=7200dd4cf518,41785,1732132095067 because future has completed 2024-11-20T19:48:19,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T19:48:19,384 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T19:48:19,384 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T19:48:19,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 5bcb6f0185137adf4e79c74b1424d03d, server=7200dd4cf518,41785,1732132095067 in 179 msec 2024-11-20T19:48:19,384 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T19:48:19,384 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T19:48:19,385 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, REOPEN/MOVE; state=CLOSED, location=7200dd4cf518,36499,1732132095222; forceNewPlan=false, retain=false 2024-11-20T19:48:19,387 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T19:48:19,388 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732132099384 2024-11-20T19:48:19,388 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T19:48:19,535 INFO [7200dd4cf518:41687 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T19:48:19,536 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5bcb6f0185137adf4e79c74b1424d03d, regionState=OPENING, regionLocation=7200dd4cf518,36499,1732132095222 2024-11-20T19:48:19,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, REOPEN/MOVE because future has completed 2024-11-20T19:48:19,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bcb6f0185137adf4e79c74b1424d03d, server=7200dd4cf518,36499,1732132095222}] 2024-11-20T19:48:19,694 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,694 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bcb6f0185137adf4e79c74b1424d03d, NAME => 'testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d.', STARTKEY => '', ENDKEY => ''} 2024-11-20T19:48:19,694 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732132097364 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,694 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:19,695 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,695 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,696 INFO [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,697 INFO [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bcb6f0185137adf4e79c74b1424d03d columnFamilyName family 2024-11-20T19:48:19,697 DEBUG [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:19,698 DEBUG [StoreFileOpener-5bcb6f0185137adf4e79c74b1424d03d-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba, delay=959 ms 2024-11-20T19:48:19,700 DEBUG [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba 2024-11-20T19:48:19,700 INFO [StoreOpener-5bcb6f0185137adf4e79c74b1424d03d-1 {}] regionserver.HStore(327): Store=5bcb6f0185137adf4e79c74b1424d03d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:19,700 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,701 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,701 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,701 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,701 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,706 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,706 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 5bcb6f0185137adf4e79c74b1424d03d; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66742303, jitterRate=-0.005462184548377991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T19:48:19,706 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,707 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5bcb6f0185137adf4e79c74b1424d03d: Running coprocessor pre-open hook at 1732132099695Writing region info on filesystem at 1732132099695Initializing all the Stores at 1732132099695Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132099695Cleaning up temporary data from old regions at 1732132099701 (+6 ms)Running coprocessor post-open hooks at 1732132099706 (+5 ms)Region opened successfully at 1732132099707 (+1 ms) 2024-11-20T19:48:19,708 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d., pid=9, masterSystemTime=1732132099691 2024-11-20T19:48:19,710 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,710 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,711 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5bcb6f0185137adf4e79c74b1424d03d, regionState=OPEN, openSeqNum=10, regionLocation=7200dd4cf518,36499,1732132095222 2024-11-20T19:48:19,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bcb6f0185137adf4e79c74b1424d03d, server=7200dd4cf518,36499,1732132095222 because future has completed 2024-11-20T19:48:19,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T19:48:19,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bcb6f0185137adf4e79c74b1424d03d, server=7200dd4cf518,36499,1732132095222 in 176 msec 2024-11-20T19:48:19,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732132097364, region=5bcb6f0185137adf4e79c74b1424d03d, REOPEN/MOVE in 521 msec 2024-11-20T19:48:19,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T19:48:19,750 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T19:48:19,750 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T19:48:19,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:19,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:19,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T19:48:19,755 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T19:48:19,755 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-20T19:48:19,756 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7200dd4cf518,41687,1732132094282 2024-11-20T19:48:19,758 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:19,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:19,758 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:19,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:19,758 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T19:48:19,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:19,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:19,759 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T19:48:19,759 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T19:48:19,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:19,759 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:19,760 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7200dd4cf518,41785,1732132095067' ***** 2024-11-20T19:48:19,760 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T19:48:19,760 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7200dd4cf518,36499,1732132095222' ***** 2024-11-20T19:48:19,760 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T19:48:19,760 INFO [RS:0;7200dd4cf518:41785 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T19:48:19,760 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:19,760 INFO [RS:1;7200dd4cf518:36499 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T19:48:19,760 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:19,760 INFO [RS:1;7200dd4cf518:36499 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/bucket.persistence 2024-11-20T19:48:19,760 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T19:48:19,761 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T19:48:19,761 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T19:48:19,761 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T19:48:19,761 INFO [RS:1;7200dd4cf518:36499 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T19:48:19,761 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T19:48:19,761 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T19:48:19,761 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T19:48:19,761 INFO [RS:0;7200dd4cf518:41785 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/bucket.persistence 2024-11-20T19:48:19,761 INFO [RS:0;7200dd4cf518:41785 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T19:48:19,762 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T19:48:19,771 DEBUG [RS:1;7200dd4cf518:36499 {}] bucket.BucketCache(1487): Thread RS:1;7200dd4cf518:36499 started persisting bucket cache to file 2024-11-20T19:48:19,771 DEBUG [RS:1;7200dd4cf518:36499 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T19:48:19,771 DEBUG [RS:1;7200dd4cf518:36499 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T19:48:19,772 DEBUG [RS:0;7200dd4cf518:41785 {}] bucket.BucketCache(1487): Thread RS:0;7200dd4cf518:41785 started persisting bucket cache to file 2024-11-20T19:48:19,772 DEBUG [RS:0;7200dd4cf518:41785 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T19:48:19,772 DEBUG [RS:0;7200dd4cf518:41785 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T19:48:19,775 DEBUG [RS:0;7200dd4cf518:41785 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T19:48:19,775 DEBUG [RS:1;7200dd4cf518:36499 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T19:48:19,775 DEBUG [RS:0;7200dd4cf518:41785 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732132099772 2024-11-20T19:48:19,775 DEBUG [RS:1;7200dd4cf518:36499 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732132099771 2024-11-20T19:48:19,775 DEBUG [RS:1;7200dd4cf518:36499 {}] bucket.BucketCache(1509): Thread RS:1;7200dd4cf518:36499 finished persisting bucket cache to file, renaming 2024-11-20T19:48:19,775 DEBUG [RS:0;7200dd4cf518:41785 {}] bucket.BucketCache(1509): Thread RS:0;7200dd4cf518:41785 finished persisting bucket cache to file, renaming 2024-11-20T19:48:19,775 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T19:48:19,776 INFO [RS:1;7200dd4cf518:36499 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T19:48:19,776 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T19:48:19,776 INFO [RS:1;7200dd4cf518:36499 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T19:48:19,776 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(3091): Received CLOSE for 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,776 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T19:48:19,776 INFO [RS:0;7200dd4cf518:41785 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T19:48:19,776 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T19:48:19,776 INFO [RS:0;7200dd4cf518:41785 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T19:48:19,777 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(959): stopping server 7200dd4cf518,41785,1732132095067 2024-11-20T19:48:19,777 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(959): stopping server 7200dd4cf518,36499,1732132095222 2024-11-20T19:48:19,777 INFO [RS:0;7200dd4cf518:41785 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T19:48:19,777 INFO [RS:1;7200dd4cf518:36499 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T19:48:19,777 INFO [RS:0;7200dd4cf518:41785 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7200dd4cf518:41785. 2024-11-20T19:48:19,777 INFO [RS:1;7200dd4cf518:36499 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7200dd4cf518:36499. 2024-11-20T19:48:19,777 DEBUG [RS:0;7200dd4cf518:41785 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T19:48:19,777 DEBUG [RS:0;7200dd4cf518:41785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:19,777 DEBUG [RS:1;7200dd4cf518:36499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T19:48:19,777 DEBUG [RS:1;7200dd4cf518:36499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:19,777 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(976): stopping server 7200dd4cf518,41785,1732132095067; all regions closed. 2024-11-20T19:48:19,777 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T19:48:19,777 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T19:48:19,777 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T19:48:19,777 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T19:48:19,777 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T19:48:19,778 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5bcb6f0185137adf4e79c74b1424d03d, disabling compactions & flushes 2024-11-20T19:48:19,778 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,778 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,778 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. after waiting 0 ms 2024-11-20T19:48:19,778 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T19:48:19,778 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,778 DEBUG [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 5bcb6f0185137adf4e79c74b1424d03d=testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d.} 2024-11-20T19:48:19,778 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T19:48:19,778 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T19:48:19,778 DEBUG [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5bcb6f0185137adf4e79c74b1424d03d 2024-11-20T19:48:19,778 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T19:48:19,778 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T19:48:19,779 DEBUG [StoreFileCloser-5bcb6f0185137adf4e79c74b1424d03d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/family/e13cbf40635c45888c0fbe0a4b519bba 2024-11-20T19:48:19,779 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T19:48:19,779 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T19:48:19,779 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-20T19:48:19,781 DEBUG [RS:0;7200dd4cf518:41785 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs 2024-11-20T19:48:19,782 INFO [RS:0;7200dd4cf518:41785 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7200dd4cf518%2C41785%2C1732132095067:(num 1732132096559) 2024-11-20T19:48:19,782 DEBUG [RS:0;7200dd4cf518:41785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:19,782 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T19:48:19,782 INFO [RS:0;7200dd4cf518:41785 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T19:48:19,782 INFO [RS:0;7200dd4cf518:41785 {}] hbase.ChoreService(370): Chore service for: regionserver/7200dd4cf518:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T19:48:19,782 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T19:48:19,782 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T19:48:19,782 INFO [regionserver/7200dd4cf518:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T19:48:19,782 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T19:48:19,783 INFO [RS:0;7200dd4cf518:41785 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T19:48:19,783 INFO [RS:0;7200dd4cf518:41785 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41785 2024-11-20T19:48:19,786 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnRegionMove1732132097364/5bcb6f0185137adf4e79c74b1424d03d/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T19:48:19,787 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,787 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5bcb6f0185137adf4e79c74b1424d03d: Waiting for close lock at 1732132099777Running coprocessor pre-close hooks at 1732132099777Disabling compacts and flushes for region at 1732132099778 (+1 ms)Disabling writes for close at 1732132099778Writing region close event to WAL at 1732132099779 (+1 ms)Running coprocessor post-close hooks at 1732132099787 (+8 ms)Closed at 1732132099787 2024-11-20T19:48:19,787 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d. 2024-11-20T19:48:19,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7200dd4cf518,41785,1732132095067 2024-11-20T19:48:19,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T19:48:19,787 INFO [RS:0;7200dd4cf518:41785 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T19:48:19,790 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7200dd4cf518,41785,1732132095067] 2024-11-20T19:48:19,790 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4de2a96120a54a1b9fc0f4a92718fcc5 2024-11-20T19:48:19,791 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7200dd4cf518,41785,1732132095067 already deleted, retry=false 2024-11-20T19:48:19,792 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7200dd4cf518,41785,1732132095067 expired; onlineServers=1 2024-11-20T19:48:19,801 INFO [regionserver/7200dd4cf518:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T19:48:19,801 INFO [regionserver/7200dd4cf518:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T19:48:19,807 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/info/4de2a96120a54a1b9fc0f4a92718fcc5 is 205, key is testBlockEvictionOnRegionMove1732132097364,,1732132097380.5bcb6f0185137adf4e79c74b1424d03d./info:regioninfo/1732132099711/Put/seqid=0 2024-11-20T19:48:19,808 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/info/4de2a96120a54a1b9fc0f4a92718fcc5 2024-11-20T19:48:19,817 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 707d7ec4c88b40b790102df22e189f62 2024-11-20T19:48:19,832 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/ns/707d7ec4c88b40b790102df22e189f62 is 43, key is default/ns:d/1732132097129/Put/seqid=0 2024-11-20T19:48:19,832 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/ns/707d7ec4c88b40b790102df22e189f62 2024-11-20T19:48:19,842 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9f0a2342ddcf45f080e827cd99e1f03d 2024-11-20T19:48:19,858 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/table/9f0a2342ddcf45f080e827cd99e1f03d is 78, key is testBlockEvictionOnRegionMove1732132097364/table:state/1732132097796/Put/seqid=0 2024-11-20T19:48:19,859 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/table/9f0a2342ddcf45f080e827cd99e1f03d 2024-11-20T19:48:19,862 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/info/4de2a96120a54a1b9fc0f4a92718fcc5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/4de2a96120a54a1b9fc0f4a92718fcc5 2024-11-20T19:48:19,864 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/4de2a96120a54a1b9fc0f4a92718fcc5, delay=982 ms 2024-11-20T19:48:19,865 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/4de2a96120a54a1b9fc0f4a92718fcc5, entries=15, sequenceid=15, filesize=7.6 K 2024-11-20T19:48:19,866 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/ns/707d7ec4c88b40b790102df22e189f62 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/707d7ec4c88b40b790102df22e189f62 2024-11-20T19:48:19,867 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/707d7ec4c88b40b790102df22e189f62, delay=926 ms 2024-11-20T19:48:19,868 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/707d7ec4c88b40b790102df22e189f62, entries=2, sequenceid=15, filesize=5.0 K 2024-11-20T19:48:19,869 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/table/9f0a2342ddcf45f080e827cd99e1f03d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/9f0a2342ddcf45f080e827cd99e1f03d 2024-11-20T19:48:19,870 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/9f0a2342ddcf45f080e827cd99e1f03d, delay=967 ms 2024-11-20T19:48:19,871 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/9f0a2342ddcf45f080e827cd99e1f03d, entries=2, sequenceid=15, filesize=5.3 K 2024-11-20T19:48:19,872 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false 2024-11-20T19:48:19,872 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T19:48:19,873 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/4de2a96120a54a1b9fc0f4a92718fcc5 2024-11-20T19:48:19,874 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/707d7ec4c88b40b790102df22e189f62 2024-11-20T19:48:19,875 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/9f0a2342ddcf45f080e827cd99e1f03d 2024-11-20T19:48:19,879 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T19:48:19,880 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T19:48:19,880 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T19:48:19,880 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732132099778Running coprocessor pre-close hooks at 1732132099778Disabling compacts and flushes for region at 1732132099778Disabling writes for close at 1732132099779 (+1 ms)Obtaining lock to block concurrent updates at 1732132099779Preparing flush snapshotting stores in 1588230740 at 1732132099779Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732132099780 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732132099780Flushing 1588230740/info: creating writer at 1732132099780Flushing 1588230740/info: appending metadata at 1732132099806 (+26 ms)Flushing 1588230740/info: closing flushed file at 1732132099806Flushing 1588230740/ns: creating writer at 1732132099810 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732132099831 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732132099831Flushing 1588230740/table: creating writer at 1732132099835 (+4 ms)Flushing 1588230740/table: appending metadata at 1732132099857 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732132099857Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@170318b3: reopening flushed file at 1732132099862 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10daa281: reopening flushed file at 1732132099865 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b3db43: reopening flushed file at 1732132099868 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false at 1732132099872 (+4 ms)Writing region close event to WAL at 1732132099875 (+3 ms)Running coprocessor post-close hooks at 1732132099879 (+4 ms)Closed at 1732132099880 (+1 ms) 2024-11-20T19:48:19,881 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T19:48:19,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:19,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41785-0x10136ec2a990001, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:19,890 INFO [RS:0;7200dd4cf518:41785 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T19:48:19,891 INFO [RS:0;7200dd4cf518:41785 {}] regionserver.HRegionServer(1031): Exiting; stopping=7200dd4cf518,41785,1732132095067; zookeeper connection closed. 2024-11-20T19:48:19,891 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@490a0336 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@490a0336 2024-11-20T19:48:19,979 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(976): stopping server 7200dd4cf518,36499,1732132095222; all regions closed. 2024-11-20T19:48:19,981 DEBUG [RS:1;7200dd4cf518:36499 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs 2024-11-20T19:48:19,981 INFO [RS:1;7200dd4cf518:36499 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7200dd4cf518%2C36499%2C1732132095222.meta:.meta(num 1732132096994) 2024-11-20T19:48:19,983 DEBUG [RS:1;7200dd4cf518:36499 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs 2024-11-20T19:48:19,983 INFO [RS:1;7200dd4cf518:36499 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7200dd4cf518%2C36499%2C1732132095222:(num 1732132096559) 2024-11-20T19:48:19,983 DEBUG [RS:1;7200dd4cf518:36499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:19,983 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T19:48:19,983 INFO [RS:1;7200dd4cf518:36499 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T19:48:19,983 INFO [RS:1;7200dd4cf518:36499 {}] hbase.ChoreService(370): Chore service for: regionserver/7200dd4cf518:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T19:48:19,983 INFO [RS:1;7200dd4cf518:36499 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T19:48:19,983 INFO [regionserver/7200dd4cf518:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T19:48:19,984 INFO [RS:1;7200dd4cf518:36499 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36499 2024-11-20T19:48:19,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T19:48:19,986 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7200dd4cf518,36499,1732132095222 2024-11-20T19:48:19,986 INFO [RS:1;7200dd4cf518:36499 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T19:48:19,988 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7200dd4cf518,36499,1732132095222] 2024-11-20T19:48:19,990 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7200dd4cf518,36499,1732132095222 already deleted, retry=false 2024-11-20T19:48:19,990 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7200dd4cf518,36499,1732132095222 expired; onlineServers=0 2024-11-20T19:48:19,990 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7200dd4cf518,41687,1732132094282' ***** 2024-11-20T19:48:19,990 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T19:48:19,990 INFO [M:0;7200dd4cf518:41687 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T19:48:19,990 INFO [M:0;7200dd4cf518:41687 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T19:48:19,990 DEBUG [M:0;7200dd4cf518:41687 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T19:48:19,991 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T19:48:19,991 DEBUG [M:0;7200dd4cf518:41687 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T19:48:19,991 DEBUG [master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.large.0-1732132096250 {}] cleaner.HFileCleaner(306): Exit Thread[master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.large.0-1732132096250,5,FailOnTimeoutGroup] 2024-11-20T19:48:19,991 DEBUG [master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.small.0-1732132096252 {}] cleaner.HFileCleaner(306): Exit Thread[master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.small.0-1732132096252,5,FailOnTimeoutGroup] 2024-11-20T19:48:19,991 INFO [M:0;7200dd4cf518:41687 {}] hbase.ChoreService(370): Chore service for: master/7200dd4cf518:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T19:48:19,991 INFO [M:0;7200dd4cf518:41687 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T19:48:19,991 DEBUG [M:0;7200dd4cf518:41687 {}] master.HMaster(1795): Stopping service threads 2024-11-20T19:48:19,991 INFO [M:0;7200dd4cf518:41687 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T19:48:19,992 INFO [M:0;7200dd4cf518:41687 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T19:48:19,992 INFO [M:0;7200dd4cf518:41687 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T19:48:19,992 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T19:48:19,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T19:48:19,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:19,993 DEBUG [M:0;7200dd4cf518:41687 {}] zookeeper.ZKUtil(347): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T19:48:19,993 WARN [M:0;7200dd4cf518:41687 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T19:48:19,994 INFO [M:0;7200dd4cf518:41687 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/.lastflushedseqids 2024-11-20T19:48:20,017 INFO [M:0;7200dd4cf518:41687 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T19:48:20,017 INFO [M:0;7200dd4cf518:41687 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T19:48:20,018 DEBUG [M:0;7200dd4cf518:41687 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T19:48:20,018 INFO [M:0;7200dd4cf518:41687 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:20,018 DEBUG [M:0;7200dd4cf518:41687 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:20,018 DEBUG [M:0;7200dd4cf518:41687 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T19:48:20,018 DEBUG [M:0;7200dd4cf518:41687 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:20,018 INFO [M:0;7200dd4cf518:41687 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-20T19:48:20,029 DEBUG [M:0;7200dd4cf518:41687 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ad4d63df2a1542eda0035fb2d9f64b50 2024-11-20T19:48:20,043 DEBUG [M:0;7200dd4cf518:41687 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ad4d63df2a1542eda0035fb2d9f64b50 is 82, key is hbase:meta,,1/info:regioninfo/1732132097063/Put/seqid=0 2024-11-20T19:48:20,044 INFO [M:0;7200dd4cf518:41687 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ad4d63df2a1542eda0035fb2d9f64b50 2024-11-20T19:48:20,052 DEBUG [M:0;7200dd4cf518:41687 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e0f9af8f90f34500b32744bbca9eb358 2024-11-20T19:48:20,066 DEBUG [M:0;7200dd4cf518:41687 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0f9af8f90f34500b32744bbca9eb358 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732132097800/Put/seqid=0 2024-11-20T19:48:20,067 INFO [M:0;7200dd4cf518:41687 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0f9af8f90f34500b32744bbca9eb358 2024-11-20T19:48:20,075 DEBUG [M:0;7200dd4cf518:41687 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ae344e3ba298402c80d1051323b8e42c 2024-11-20T19:48:20,089 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:20,089 INFO [RS:1;7200dd4cf518:36499 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T19:48:20,089 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36499-0x10136ec2a990002, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:20,089 INFO [RS:1;7200dd4cf518:36499 {}] regionserver.HRegionServer(1031): Exiting; stopping=7200dd4cf518,36499,1732132095222; zookeeper connection closed. 2024-11-20T19:48:20,089 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6b2019b2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6b2019b2 2024-11-20T19:48:20,089 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-20T19:48:20,095 DEBUG [M:0;7200dd4cf518:41687 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ae344e3ba298402c80d1051323b8e42c is 69, key is 7200dd4cf518,36499,1732132095222/rs:state/1732132096308/Put/seqid=0 2024-11-20T19:48:20,096 INFO [M:0;7200dd4cf518:41687 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ae344e3ba298402c80d1051323b8e42c 2024-11-20T19:48:20,098 DEBUG [M:0;7200dd4cf518:41687 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ad4d63df2a1542eda0035fb2d9f64b50 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ad4d63df2a1542eda0035fb2d9f64b50 2024-11-20T19:48:20,101 INFO [M:0;7200dd4cf518:41687 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ad4d63df2a1542eda0035fb2d9f64b50, entries=8, sequenceid=84, filesize=5.5 K 2024-11-20T19:48:20,101 DEBUG [M:0;7200dd4cf518:41687 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0f9af8f90f34500b32744bbca9eb358 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e0f9af8f90f34500b32744bbca9eb358 2024-11-20T19:48:20,102 INFO [M:0;7200dd4cf518:41687 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e0f9af8f90f34500b32744bbca9eb358, entries=9, sequenceid=84, filesize=6.5 K 2024-11-20T19:48:20,103 DEBUG [M:0;7200dd4cf518:41687 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ae344e3ba298402c80d1051323b8e42c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ae344e3ba298402c80d1051323b8e42c 2024-11-20T19:48:20,104 INFO [M:0;7200dd4cf518:41687 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ae344e3ba298402c80d1051323b8e42c, entries=2, sequenceid=84, filesize=5.1 K 2024-11-20T19:48:20,105 INFO [M:0;7200dd4cf518:41687 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=84, compaction requested=false 2024-11-20T19:48:20,107 INFO [M:0;7200dd4cf518:41687 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:20,107 DEBUG [M:0;7200dd4cf518:41687 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732132100018Disabling compacts and flushes for region at 1732132100018Disabling writes for close at 1732132100018Obtaining lock to block concurrent updates at 1732132100018Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732132100018Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732132100019 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732132100019Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732132100020 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732132100043 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732132100043Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732132100046 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732132100065 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732132100066 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732132100069 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732132100094 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732132100094Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13d25118: reopening flushed file at 1732132100098 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a8d19f4: reopening flushed file at 1732132100101 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aa0f3b9: reopening flushed file at 1732132100103 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=84, compaction requested=false at 1732132100105 (+2 ms)Writing region close event to WAL at 1732132100107 (+2 ms)Closed at 1732132100107 2024-11-20T19:48:20,109 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T19:48:20,109 INFO [M:0;7200dd4cf518:41687 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T19:48:20,110 INFO [M:0;7200dd4cf518:41687 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41687 2024-11-20T19:48:20,110 INFO [M:0;7200dd4cf518:41687 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T19:48:20,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:20,212 INFO [M:0;7200dd4cf518:41687 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T19:48:20,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41687-0x10136ec2a990000, quorum=127.0.0.1:50782, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:20,216 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T19:48:20,237 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T19:48:20,237 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:20,246 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/7200dd4cf518:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732132099867 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/7200dd4cf518:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732132099698 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/7200dd4cf518:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/7200dd4cf518:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732132099864 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732132098173 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=257 (was 278), ProcessCount=11 (was 11), AvailableMemoryMB=5354 (was 5708) 2024-11-20T19:48:20,252 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=257, ProcessCount=11, AvailableMemoryMB=5353 2024-11-20T19:48:20,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/hadoop-log-dir so I do NOT create it in target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20 2024-11-20T19:48:20,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/hadoop-tmp-dir so I do NOT create it in target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20 2024-11-20T19:48:20,252 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:20,255 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/cluster_ccde0398-7af5-f029-a72d-0aad0dcbfa4f, deleteOnExit=true 2024-11-20T19:48:20,258 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/cluster_ccde0398-7af5-f029-a72d-0aad0dcbfa4f/zookeeper_0, clientPort=63373, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/cluster_ccde0398-7af5-f029-a72d-0aad0dcbfa4f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/cluster_ccde0398-7af5-f029-a72d-0aad0dcbfa4f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T19:48:20,259 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63373 2024-11-20T19:48:20,270 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 with version=8 2024-11-20T19:48:20,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/hbase-staging 2024-11-20T19:48:20,273 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7200dd4cf518:0 server-side Connection retries=45 2024-11-20T19:48:20,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:20,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:20,274 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T19:48:20,274 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:20,274 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T19:48:20,274 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T19:48:20,274 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T19:48:20,275 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34433 2024-11-20T19:48:20,276 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34433 connecting to ZooKeeper ensemble=127.0.0.1:63373 2024-11-20T19:48:20,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:344330x0, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T19:48:20,285 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34433-0x10136ec43310000 connected 2024-11-20T19:48:20,301 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:20,302 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:20,304 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:20,304 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927, hbase.cluster.distributed=false 2024-11-20T19:48:20,306 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T19:48:20,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34433 2024-11-20T19:48:20,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34433 2024-11-20T19:48:20,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34433 2024-11-20T19:48:20,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34433 2024-11-20T19:48:20,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34433 2024-11-20T19:48:20,323 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7200dd4cf518:0 server-side Connection retries=45 2024-11-20T19:48:20,323 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:20,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:20,324 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T19:48:20,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:20,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T19:48:20,324 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T19:48:20,324 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T19:48:20,325 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37581 2024-11-20T19:48:20,326 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37581 connecting to ZooKeeper ensemble=127.0.0.1:63373 2024-11-20T19:48:20,326 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:20,326 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:20,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:375810x0, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T19:48:20,331 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37581-0x10136ec43310001 connected 2024-11-20T19:48:20,331 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:20,332 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T19:48:20,333 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.cache 2024-11-20T19:48:20,333 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T19:48:20,333 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T19:48:20,333 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T19:48:20,333 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T19:48:20,333 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T19:48:20,334 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T19:48:20,334 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T19:48:20,334 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.persistence is complete. 2024-11-20T19:48:20,334 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T19:48:20,335 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T19:48:20,336 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T19:48:20,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37581 2024-11-20T19:48:20,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37581 2024-11-20T19:48:20,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37581 2024-11-20T19:48:20,342 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37581 2024-11-20T19:48:20,342 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37581 2024-11-20T19:48:20,358 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7200dd4cf518:0 server-side Connection retries=45 2024-11-20T19:48:20,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:20,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:20,358 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T19:48:20,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:20,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T19:48:20,358 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T19:48:20,358 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T19:48:20,359 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36943 2024-11-20T19:48:20,361 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36943 connecting to ZooKeeper ensemble=127.0.0.1:63373 2024-11-20T19:48:20,361 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:20,361 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:20,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:369430x0, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T19:48:20,365 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36943-0x10136ec43310002 connected 2024-11-20T19:48:20,365 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:20,366 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T19:48:20,366 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.cache 2024-11-20T19:48:20,366 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T19:48:20,367 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T19:48:20,367 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T19:48:20,367 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T19:48:20,367 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T19:48:20,367 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T19:48:20,367 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T19:48:20,368 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.persistence is complete. 2024-11-20T19:48:20,368 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T19:48:20,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T19:48:20,370 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T19:48:20,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36943 2024-11-20T19:48:20,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36943 2024-11-20T19:48:20,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36943 2024-11-20T19:48:20,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36943 2024-11-20T19:48:20,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36943 2024-11-20T19:48:20,395 DEBUG [M:0;7200dd4cf518:34433 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7200dd4cf518:34433 2024-11-20T19:48:20,396 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7200dd4cf518,34433,1732132100273 2024-11-20T19:48:20,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:20,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:20,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:20,399 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7200dd4cf518,34433,1732132100273 2024-11-20T19:48:20,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T19:48:20,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T19:48:20,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,402 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T19:48:20,403 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7200dd4cf518,34433,1732132100273 from backup master directory 2024-11-20T19:48:20,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7200dd4cf518,34433,1732132100273 2024-11-20T19:48:20,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:20,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:20,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T19:48:20,405 WARN [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T19:48:20,405 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7200dd4cf518,34433,1732132100273 2024-11-20T19:48:20,407 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/hbase.id] with ID: 90a8ff9e-87a4-4858-9977-8644c0a34876 2024-11-20T19:48:20,407 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/.tmp/hbase.id 2024-11-20T19:48:20,412 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/hbase.id] 2024-11-20T19:48:20,461 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:20,461 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T19:48:20,463 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-20T19:48:20,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,470 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T19:48:20,472 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T19:48:20,472 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T19:48:20,493 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store 2024-11-20T19:48:20,499 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:20,499 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T19:48:20,499 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:20,499 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:20,499 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T19:48:20,499 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:20,500 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:20,500 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732132100499Disabling compacts and flushes for region at 1732132100499Disabling writes for close at 1732132100499Writing region close event to WAL at 1732132100500 (+1 ms)Closed at 1732132100500 2024-11-20T19:48:20,500 WARN [master/7200dd4cf518:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/.initializing 2024-11-20T19:48:20,500 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/WALs/7200dd4cf518,34433,1732132100273 2024-11-20T19:48:20,505 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T19:48:20,511 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7200dd4cf518%2C34433%2C1732132100273, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/WALs/7200dd4cf518,34433,1732132100273, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/oldWALs, maxLogs=10 2024-11-20T19:48:20,529 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/WALs/7200dd4cf518,34433,1732132100273/7200dd4cf518%2C34433%2C1732132100273.1732132100512 2024-11-20T19:48:20,530 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T19:48:20,530 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T19:48:20,530 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:20,530 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,530 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T19:48:20,537 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:20,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:20,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T19:48:20,541 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:20,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:20,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T19:48:20,545 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:20,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:20,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,549 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T19:48:20,549 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:20,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:20,550 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,550 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,550 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,551 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,551 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,551 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T19:48:20,551 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T19:48:20,563 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T19:48:20,563 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71340155, jitterRate=0.06305114924907684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T19:48:20,565 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732132100530Initializing all the Stores at 1732132100531 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132100531Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132100531Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132100531Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132100531Cleaning up temporary data from old regions at 1732132100551 (+20 ms)Region opened successfully at 1732132100564 (+13 ms) 2024-11-20T19:48:20,565 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T19:48:20,570 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33068bbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7200dd4cf518/172.17.0.2:0 2024-11-20T19:48:20,571 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T19:48:20,572 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T19:48:20,572 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T19:48:20,572 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T19:48:20,572 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T19:48:20,573 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-20T19:48:20,573 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T19:48:20,575 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T19:48:20,577 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T19:48:20,578 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T19:48:20,578 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T19:48:20,579 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T19:48:20,581 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T19:48:20,581 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T19:48:20,582 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T19:48:20,585 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T19:48:20,586 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T19:48:20,587 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T19:48:20,590 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T19:48:20,591 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T19:48:20,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:20,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:20,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:20,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,594 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7200dd4cf518,34433,1732132100273, sessionid=0x10136ec43310000, setting cluster-up flag (Was=false) 2024-11-20T19:48:20,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,604 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T19:48:20,606 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7200dd4cf518,34433,1732132100273 2024-11-20T19:48:20,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:20,615 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T19:48:20,616 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7200dd4cf518,34433,1732132100273 2024-11-20T19:48:20,617 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T19:48:20,619 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T19:48:20,619 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T19:48:20,619 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T19:48:20,620 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7200dd4cf518,34433,1732132100273 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T19:48:20,622 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7200dd4cf518:0, corePoolSize=5, maxPoolSize=5 2024-11-20T19:48:20,622 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7200dd4cf518:0, corePoolSize=5, maxPoolSize=5 2024-11-20T19:48:20,622 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7200dd4cf518:0, corePoolSize=5, maxPoolSize=5 2024-11-20T19:48:20,623 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7200dd4cf518:0, corePoolSize=5, maxPoolSize=5 2024-11-20T19:48:20,623 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7200dd4cf518:0, corePoolSize=10, maxPoolSize=10 2024-11-20T19:48:20,623 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,623 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7200dd4cf518:0, corePoolSize=2, maxPoolSize=2 2024-11-20T19:48:20,623 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,625 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732132130625 2024-11-20T19:48:20,625 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T19:48:20,625 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T19:48:20,625 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T19:48:20,625 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T19:48:20,625 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T19:48:20,626 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T19:48:20,626 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T19:48:20,626 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T19:48:20,626 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:20,626 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,626 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T19:48:20,626 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T19:48:20,626 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T19:48:20,627 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T19:48:20,627 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T19:48:20,627 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T19:48:20,627 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.large.0-1732132100627,5,FailOnTimeoutGroup] 2024-11-20T19:48:20,627 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.small.0-1732132100627,5,FailOnTimeoutGroup] 2024-11-20T19:48:20,627 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,628 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T19:48:20,628 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,628 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,645 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T19:48:20,645 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 2024-11-20T19:48:20,651 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:20,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T19:48:20,656 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T19:48:20,656 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:20,656 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:20,656 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T19:48:20,659 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T19:48:20,660 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:20,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:20,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T19:48:20,663 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T19:48:20,663 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:20,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:20,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T19:48:20,667 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T19:48:20,667 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:20,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:20,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T19:48:20,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740 2024-11-20T19:48:20,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740 2024-11-20T19:48:20,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T19:48:20,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T19:48:20,668 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T19:48:20,669 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T19:48:20,675 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T19:48:20,675 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65161355, jitterRate=-0.029020145535469055}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T19:48:20,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732132100652Initializing all the Stores at 1732132100652Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132100652Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132100652Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132100652Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132100652Cleaning up temporary data from old regions at 1732132100668 (+16 ms)Region opened successfully at 1732132100676 (+8 ms) 2024-11-20T19:48:20,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T19:48:20,676 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T19:48:20,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T19:48:20,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T19:48:20,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T19:48:20,676 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T19:48:20,677 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732132100676Disabling compacts and flushes for region at 1732132100676Disabling writes for close at 1732132100676Writing region close event to WAL at 1732132100676Closed at 1732132100676 2024-11-20T19:48:20,678 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T19:48:20,678 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T19:48:20,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T19:48:20,679 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(746): ClusterId : 90a8ff9e-87a4-4858-9977-8644c0a34876 2024-11-20T19:48:20,679 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(746): ClusterId : 90a8ff9e-87a4-4858-9977-8644c0a34876 2024-11-20T19:48:20,679 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T19:48:20,679 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T19:48:20,679 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T19:48:20,680 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T19:48:20,682 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T19:48:20,682 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T19:48:20,682 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T19:48:20,682 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T19:48:20,684 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T19:48:20,684 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T19:48:20,685 DEBUG [RS:0;7200dd4cf518:37581 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81264ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7200dd4cf518/172.17.0.2:0 2024-11-20T19:48:20,685 DEBUG [RS:1;7200dd4cf518:36943 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16eefe87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7200dd4cf518/172.17.0.2:0 2024-11-20T19:48:20,697 DEBUG [RS:0;7200dd4cf518:37581 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7200dd4cf518:37581 2024-11-20T19:48:20,697 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T19:48:20,697 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T19:48:20,697 DEBUG [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T19:48:20,698 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(2659): reportForDuty to master=7200dd4cf518,34433,1732132100273 with port=37581, startcode=1732132100323 2024-11-20T19:48:20,698 DEBUG [RS:0;7200dd4cf518:37581 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T19:48:20,700 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46381, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T19:48:20,701 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7200dd4cf518,37581,1732132100323 2024-11-20T19:48:20,701 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34433 {}] master.ServerManager(517): Registering regionserver=7200dd4cf518,37581,1732132100323 2024-11-20T19:48:20,703 DEBUG [RS:1;7200dd4cf518:36943 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7200dd4cf518:36943 2024-11-20T19:48:20,703 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T19:48:20,703 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T19:48:20,703 DEBUG [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T19:48:20,703 DEBUG [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 2024-11-20T19:48:20,703 DEBUG [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T19:48:20,703 DEBUG [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T19:48:20,704 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(2659): reportForDuty to master=7200dd4cf518,34433,1732132100273 with port=36943, startcode=1732132100357 2024-11-20T19:48:20,704 DEBUG [RS:1;7200dd4cf518:36943 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T19:48:20,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T19:48:20,706 DEBUG [RS:0;7200dd4cf518:37581 {}] zookeeper.ZKUtil(111): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7200dd4cf518,37581,1732132100323 2024-11-20T19:48:20,706 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35913, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T19:48:20,707 WARN [RS:0;7200dd4cf518:37581 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T19:48:20,707 INFO [RS:0;7200dd4cf518:37581 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T19:48:20,707 DEBUG [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,37581,1732132100323 2024-11-20T19:48:20,707 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7200dd4cf518,36943,1732132100357 2024-11-20T19:48:20,707 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7200dd4cf518,37581,1732132100323] 2024-11-20T19:48:20,707 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34433 {}] master.ServerManager(517): Registering regionserver=7200dd4cf518,36943,1732132100357 2024-11-20T19:48:20,709 DEBUG [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 2024-11-20T19:48:20,709 DEBUG [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T19:48:20,709 DEBUG [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T19:48:20,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T19:48:20,716 DEBUG [RS:1;7200dd4cf518:36943 {}] zookeeper.ZKUtil(111): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7200dd4cf518,36943,1732132100357 2024-11-20T19:48:20,716 WARN [RS:1;7200dd4cf518:36943 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T19:48:20,717 INFO [RS:1;7200dd4cf518:36943 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T19:48:20,717 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7200dd4cf518,36943,1732132100357] 2024-11-20T19:48:20,717 DEBUG [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36943,1732132100357 2024-11-20T19:48:20,717 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T19:48:20,721 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T19:48:20,721 INFO [RS:0;7200dd4cf518:37581 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T19:48:20,721 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,722 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T19:48:20,722 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T19:48:20,722 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,722 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7200dd4cf518:0, corePoolSize=2, maxPoolSize=2 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7200dd4cf518:0, corePoolSize=3, maxPoolSize=3 2024-11-20T19:48:20,723 DEBUG [RS:0;7200dd4cf518:37581 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7200dd4cf518:0, corePoolSize=3, maxPoolSize=3 2024-11-20T19:48:20,724 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,724 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,724 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,724 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,724 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,724 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,37581,1732132100323-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T19:48:20,725 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T19:48:20,726 INFO [RS:1;7200dd4cf518:36943 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T19:48:20,726 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,726 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T19:48:20,727 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T19:48:20,727 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7200dd4cf518:0, corePoolSize=2, maxPoolSize=2 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,727 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,728 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:20,728 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7200dd4cf518:0, corePoolSize=3, maxPoolSize=3 2024-11-20T19:48:20,728 DEBUG [RS:1;7200dd4cf518:36943 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7200dd4cf518:0, corePoolSize=3, maxPoolSize=3 2024-11-20T19:48:20,728 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,728 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,728 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,728 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,728 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,728 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,36943,1732132100357-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T19:48:20,739 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T19:48:20,740 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,37581,1732132100323-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,740 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,740 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.Replication(171): 7200dd4cf518,37581,1732132100323 started 2024-11-20T19:48:20,745 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T19:48:20,745 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,36943,1732132100357-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,746 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,746 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.Replication(171): 7200dd4cf518,36943,1732132100357 started 2024-11-20T19:48:20,754 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,754 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(1482): Serving as 7200dd4cf518,37581,1732132100323, RpcServer on 7200dd4cf518/172.17.0.2:37581, sessionid=0x10136ec43310001 2024-11-20T19:48:20,755 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T19:48:20,755 DEBUG [RS:0;7200dd4cf518:37581 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7200dd4cf518,37581,1732132100323 2024-11-20T19:48:20,755 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7200dd4cf518,37581,1732132100323' 2024-11-20T19:48:20,755 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T19:48:20,756 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T19:48:20,756 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T19:48:20,756 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T19:48:20,756 DEBUG [RS:0;7200dd4cf518:37581 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7200dd4cf518,37581,1732132100323 2024-11-20T19:48:20,756 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7200dd4cf518,37581,1732132100323' 2024-11-20T19:48:20,757 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T19:48:20,757 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T19:48:20,757 DEBUG [RS:0;7200dd4cf518:37581 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T19:48:20,758 INFO [RS:0;7200dd4cf518:37581 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T19:48:20,758 INFO [RS:0;7200dd4cf518:37581 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T19:48:20,760 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:20,760 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(1482): Serving as 7200dd4cf518,36943,1732132100357, RpcServer on 7200dd4cf518/172.17.0.2:36943, sessionid=0x10136ec43310002 2024-11-20T19:48:20,760 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T19:48:20,760 DEBUG [RS:1;7200dd4cf518:36943 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7200dd4cf518,36943,1732132100357 2024-11-20T19:48:20,760 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7200dd4cf518,36943,1732132100357' 2024-11-20T19:48:20,760 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T19:48:20,761 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T19:48:20,761 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T19:48:20,761 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T19:48:20,761 DEBUG [RS:1;7200dd4cf518:36943 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7200dd4cf518,36943,1732132100357 2024-11-20T19:48:20,761 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7200dd4cf518,36943,1732132100357' 2024-11-20T19:48:20,761 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T19:48:20,761 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T19:48:20,762 DEBUG [RS:1;7200dd4cf518:36943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T19:48:20,762 INFO [RS:1;7200dd4cf518:36943 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T19:48:20,762 INFO [RS:1;7200dd4cf518:36943 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T19:48:20,830 WARN [7200dd4cf518:34433 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T19:48:20,858 INFO [RS:0;7200dd4cf518:37581 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T19:48:20,859 INFO [RS:0;7200dd4cf518:37581 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7200dd4cf518%2C37581%2C1732132100323, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,37581,1732132100323, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs, maxLogs=55 2024-11-20T19:48:20,863 INFO [RS:1;7200dd4cf518:36943 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T19:48:20,863 INFO [RS:1;7200dd4cf518:36943 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7200dd4cf518%2C36943%2C1732132100357, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36943,1732132100357, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs, maxLogs=55 2024-11-20T19:48:20,879 INFO [RS:0;7200dd4cf518:37581 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,37581,1732132100323/7200dd4cf518%2C37581%2C1732132100323.1732132100861 2024-11-20T19:48:20,880 DEBUG [RS:0;7200dd4cf518:37581 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T19:48:20,881 INFO [RS:1;7200dd4cf518:36943 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36943,1732132100357/7200dd4cf518%2C36943%2C1732132100357.1732132100864 2024-11-20T19:48:20,881 DEBUG [RS:1;7200dd4cf518:36943 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T19:48:21,081 DEBUG [7200dd4cf518:34433 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T19:48:21,081 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(204): Hosts are {7200dd4cf518=0} racks are {/default-rack=0} 2024-11-20T19:48:21,082 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T19:48:21,082 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T19:48:21,082 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T19:48:21,082 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T19:48:21,082 INFO [7200dd4cf518:34433 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T19:48:21,082 INFO [7200dd4cf518:34433 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T19:48:21,082 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T19:48:21,082 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7200dd4cf518,36943,1732132100357 2024-11-20T19:48:21,084 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7200dd4cf518,36943,1732132100357, state=OPENING 2024-11-20T19:48:21,085 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T19:48:21,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:21,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:21,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:21,088 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:21,088 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:21,088 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:21,088 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T19:48:21,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7200dd4cf518,36943,1732132100357}] 2024-11-20T19:48:21,242 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T19:48:21,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54387, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T19:48:21,247 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T19:48:21,247 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T19:48:21,247 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T19:48:21,248 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7200dd4cf518%2C36943%2C1732132100357.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36943,1732132100357, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs, maxLogs=55 2024-11-20T19:48:21,268 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36943,1732132100357/7200dd4cf518%2C36943%2C1732132100357.meta.1732132101249.meta 2024-11-20T19:48:21,268 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T19:48:21,269 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T19:48:21,269 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T19:48:21,269 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T19:48:21,269 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T19:48:21,269 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T19:48:21,270 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:21,270 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T19:48:21,270 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T19:48:21,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T19:48:21,272 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T19:48:21,272 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:21,272 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:21,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T19:48:21,273 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T19:48:21,273 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:21,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:21,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T19:48:21,274 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T19:48:21,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:21,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:21,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T19:48:21,276 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T19:48:21,276 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:21,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:21,276 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T19:48:21,277 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740 2024-11-20T19:48:21,277 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740 2024-11-20T19:48:21,277 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T19:48:21,277 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T19:48:21,278 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T19:48:21,278 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T19:48:21,279 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75360682, jitterRate=0.12296167016029358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T19:48:21,279 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T19:48:21,280 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732132101270Writing region info on filesystem at 1732132101270Initializing all the Stores at 1732132101270Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132101270Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132101271 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132101271Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132101271Cleaning up temporary data from old regions at 1732132101277 (+6 ms)Running coprocessor post-open hooks at 1732132101279 (+2 ms)Region opened successfully at 1732132101280 (+1 ms) 2024-11-20T19:48:21,281 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732132101241 2024-11-20T19:48:21,285 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7200dd4cf518,36943,1732132100357 2024-11-20T19:48:21,285 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T19:48:21,285 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T19:48:21,286 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7200dd4cf518,36943,1732132100357, state=OPEN 2024-11-20T19:48:21,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:21,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:21,288 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:21,288 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7200dd4cf518,36943,1732132100357 2024-11-20T19:48:21,288 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:21,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:21,288 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:21,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T19:48:21,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7200dd4cf518,36943,1732132100357 in 200 msec 2024-11-20T19:48:21,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T19:48:21,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 613 msec 2024-11-20T19:48:21,293 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T19:48:21,293 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T19:48:21,298 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T19:48:21,298 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7200dd4cf518,36943,1732132100357, seqNum=-1] 2024-11-20T19:48:21,298 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T19:48:21,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40709, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T19:48:21,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 688 msec 2024-11-20T19:48:21,306 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732132101306, completionTime=-1 2024-11-20T19:48:21,307 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T19:48:21,307 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T19:48:21,309 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T19:48:21,309 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732132161309 2024-11-20T19:48:21,310 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732132221309 2024-11-20T19:48:21,310 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-20T19:48:21,310 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T19:48:21,311 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,34433,1732132100273-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:21,311 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,34433,1732132100273-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:21,311 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,34433,1732132100273-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:21,311 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7200dd4cf518:34433, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:21,311 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:21,311 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:21,314 DEBUG [master/7200dd4cf518:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T19:48:21,317 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.911sec 2024-11-20T19:48:21,317 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T19:48:21,317 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T19:48:21,317 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T19:48:21,317 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T19:48:21,317 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T19:48:21,317 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,34433,1732132100273-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T19:48:21,317 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,34433,1732132100273-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T19:48:21,318 DEBUG [master/7200dd4cf518:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T19:48:21,318 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T19:48:21,319 INFO [master/7200dd4cf518:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,34433,1732132100273-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:21,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2129cce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T19:48:21,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7200dd4cf518,34433,-1 for getting cluster id 2024-11-20T19:48:21,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T19:48:21,381 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '90a8ff9e-87a4-4858-9977-8644c0a34876' 2024-11-20T19:48:21,382 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T19:48:21,382 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "90a8ff9e-87a4-4858-9977-8644c0a34876" 2024-11-20T19:48:21,383 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a9d5b03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T19:48:21,383 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7200dd4cf518,34433,-1] 2024-11-20T19:48:21,383 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T19:48:21,384 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:21,387 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T19:48:21,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T19:48:21,388 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T19:48:21,390 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7200dd4cf518,36943,1732132100357, seqNum=-1] 2024-11-20T19:48:21,390 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T19:48:21,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T19:48:21,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7200dd4cf518,34433,1732132100273 2024-11-20T19:48:21,398 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T19:48:21,399 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 7200dd4cf518,34433,1732132100273 2024-11-20T19:48:21,399 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@765951fb 2024-11-20T19:48:21,399 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T19:48:21,401 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T19:48:21,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732132101397', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T19:48:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732132101397 2024-11-20T19:48:21,410 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732132101397 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T19:48:21,411 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:21,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732132101397" procId is: 4 2024-11-20T19:48:21,412 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732132101397 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T19:48:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T19:48:21,428 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732132101397-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 06aec7224c925f4cd7c46e2504a5df38, NAME => 'testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732132101397', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 2024-11-20T19:48:21,440 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732132101397-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:21,441 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732132101397-pool-0 {}] regionserver.HRegion(1722): Closing 06aec7224c925f4cd7c46e2504a5df38, disabling compactions & flushes 2024-11-20T19:48:21,441 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732132101397-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:21,441 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732132101397-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:21,441 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732132101397-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. after waiting 0 ms 2024-11-20T19:48:21,441 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732132101397-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:21,441 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732132101397-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:21,441 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732132101397-pool-0 {}] regionserver.HRegion(1676): Region close journal for 06aec7224c925f4cd7c46e2504a5df38: Waiting for close lock at 1732132101441Disabling compacts and flushes for region at 1732132101441Disabling writes for close at 1732132101441Writing region close event to WAL at 1732132101441Closed at 1732132101441 2024-11-20T19:48:21,443 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732132101397 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T19:48:21,444 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732132101443"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732132101443"}]},"ts":"1732132101443"} 2024-11-20T19:48:21,447 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T19:48:21,449 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732132101397 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T19:48:21,449 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732132101397","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732132101449"}]},"ts":"1732132101449"} 2024-11-20T19:48:21,454 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732132101397, state=ENABLING in hbase:meta 2024-11-20T19:48:21,454 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {7200dd4cf518=0} racks are {/default-rack=0} 2024-11-20T19:48:21,455 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T19:48:21,455 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T19:48:21,455 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T19:48:21,455 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T19:48:21,455 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T19:48:21,455 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T19:48:21,455 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T19:48:21,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732132101397, region=06aec7224c925f4cd7c46e2504a5df38, ASSIGN}] 2024-11-20T19:48:21,457 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732132101397, region=06aec7224c925f4cd7c46e2504a5df38, ASSIGN 2024-11-20T19:48:21,459 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732132101397, region=06aec7224c925f4cd7c46e2504a5df38, ASSIGN; state=OFFLINE, location=7200dd4cf518,36943,1732132100357; forceNewPlan=false, retain=false 2024-11-20T19:48:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T19:48:21,610 INFO [7200dd4cf518:34433 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T19:48:21,610 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=06aec7224c925f4cd7c46e2504a5df38, regionState=OPENING, regionLocation=7200dd4cf518,36943,1732132100357 2024-11-20T19:48:21,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732132101397, region=06aec7224c925f4cd7c46e2504a5df38, ASSIGN because future has completed 2024-11-20T19:48:21,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06aec7224c925f4cd7c46e2504a5df38, server=7200dd4cf518,36943,1732132100357}] 2024-11-20T19:48:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T19:48:21,771 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:21,771 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 06aec7224c925f4cd7c46e2504a5df38, NAME => 'testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38.', STARTKEY => '', ENDKEY => ''} 2024-11-20T19:48:21,771 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732132101397 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,772 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:21,772 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,772 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,773 INFO [StoreOpener-06aec7224c925f4cd7c46e2504a5df38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,779 INFO [StoreOpener-06aec7224c925f4cd7c46e2504a5df38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06aec7224c925f4cd7c46e2504a5df38 columnFamilyName family 2024-11-20T19:48:21,779 DEBUG [StoreOpener-06aec7224c925f4cd7c46e2504a5df38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:21,780 INFO [StoreOpener-06aec7224c925f4cd7c46e2504a5df38-1 {}] regionserver.HStore(327): Store=06aec7224c925f4cd7c46e2504a5df38/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:21,780 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,781 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,781 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,781 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,781 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,782 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,802 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T19:48:21,803 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 06aec7224c925f4cd7c46e2504a5df38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68608951, jitterRate=0.02235303819179535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T19:48:21,803 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:21,804 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 06aec7224c925f4cd7c46e2504a5df38: Running coprocessor pre-open hook at 1732132101772Writing region info on filesystem at 1732132101772Initializing all the Stores at 1732132101772Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132101772Cleaning up temporary data from old regions at 1732132101781 (+9 ms)Running coprocessor post-open hooks at 1732132101803 (+22 ms)Region opened successfully at 1732132101804 (+1 ms) 2024-11-20T19:48:21,807 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38., pid=6, masterSystemTime=1732132101768 2024-11-20T19:48:21,811 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:21,811 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:21,811 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=06aec7224c925f4cd7c46e2504a5df38, regionState=OPEN, openSeqNum=2, regionLocation=7200dd4cf518,36943,1732132100357 2024-11-20T19:48:21,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06aec7224c925f4cd7c46e2504a5df38, server=7200dd4cf518,36943,1732132100357 because future has completed 2024-11-20T19:48:21,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T19:48:21,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 06aec7224c925f4cd7c46e2504a5df38, server=7200dd4cf518,36943,1732132100357 in 203 msec 2024-11-20T19:48:21,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T19:48:21,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732132101397, region=06aec7224c925f4cd7c46e2504a5df38, ASSIGN in 364 msec 2024-11-20T19:48:21,824 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732132101397 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T19:48:21,825 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732132101397","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732132101824"}]},"ts":"1732132101824"} 2024-11-20T19:48:21,828 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732132101397, state=ENABLED in hbase:meta 2024-11-20T19:48:21,829 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732132101397 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T19:48:21,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732132101397 in 427 msec 2024-11-20T19:48:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T19:48:22,040 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732132101397 completed 2024-11-20T19:48:22,040 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732132101397 get assigned. Timeout = 60000ms 2024-11-20T19:48:22,040 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T19:48:22,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732132101397 assigned to meta. Checking AM states. 2024-11-20T19:48:22,044 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T19:48:22,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732132101397 assigned. 2024-11-20T19:48:22,048 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732132101397', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38., hostname=7200dd4cf518,36943,1732132100357, seqNum=2] 2024-11-20T19:48:22,054 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 06aec7224c925f4cd7c46e2504a5df38 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T19:48:22,066 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a809000669484d2a8a16589a6b0911bd 2024-11-20T19:48:22,079 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/.tmp/family/a809000669484d2a8a16589a6b0911bd is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T19:48:22,079 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/.tmp/family/a809000669484d2a8a16589a6b0911bd 2024-11-20T19:48:22,083 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/.tmp/family/a809000669484d2a8a16589a6b0911bd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd 2024-11-20T19:48:22,085 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd, delay=905 ms 2024-11-20T19:48:22,087 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T19:48:22,089 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 06aec7224c925f4cd7c46e2504a5df38 in 34ms, sequenceid=6, compaction requested=false 2024-11-20T19:48:22,089 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732132101397' 2024-11-20T19:48:22,090 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 06aec7224c925f4cd7c46e2504a5df38: 2024-11-20T19:48:22,532 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T19:48:22,991 DEBUG [hfile-prefetch-1732132099698 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a809000669484d2a8a16589a6b0911bd_0. 2024-11-20T19:48:22,992 DEBUG [hfile-prefetch-1732132099698 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a809000669484d2a8a16589a6b0911bd_117. 2024-11-20T19:48:22,993 DEBUG [hfile-prefetch-1732132099698 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd, with total blocks 2, and data blocks 1 2024-11-20T19:48:22,993 DEBUG [hfile-prefetch-1732132099698 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T19:48:22,993 DEBUG [hfile-prefetch-1732132099698 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T19:48:22,993 DEBUG [hfile-prefetch-1732132099698 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd. Will sleep 100ms and try the verification again. 2024-11-20T19:48:23,035 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T19:48:23,035 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T19:48:23,036 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T19:48:23,036 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T19:48:23,039 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T19:48:23,039 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732132103035 2024-11-20T19:48:23,039 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T19:48:23,090 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;7200dd4cf518:36943,5,FailOnTimeoutGroup] 2024-11-20T19:48:23,090 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7200dd4cf518,36943,1732132100357' ***** 2024-11-20T19:48:23,090 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-20T19:48:23,091 INFO [RS:1;7200dd4cf518:36943 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T19:48:23,091 INFO [RS:1;7200dd4cf518:36943 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.persistence 2024-11-20T19:48:23,091 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T19:48:23,091 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T19:48:23,091 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T19:48:23,091 INFO [RS:1;7200dd4cf518:36943 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T19:48:23,091 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T19:48:23,093 DEBUG [hfile-prefetch-1732132099698 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd, with total blocks 2, and data blocks 1 2024-11-20T19:48:23,093 DEBUG [hfile-prefetch-1732132099698 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-20T19:48:23,093 DEBUG [hfile-prefetch-1732132099698 {}] bucket.BucketCache(2293): found block for file a809000669484d2a8a16589a6b0911bd in the backing map. Acquiring read lock for offset 0 2024-11-20T19:48:23,093 DEBUG [hfile-prefetch-1732132099698 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd has now been fully cached. 2024-11-20T19:48:23,093 DEBUG [hfile-prefetch-1732132099698 {}] hfile.PrefetchExecutor(121): Prefetch completed for a809000669484d2a8a16589a6b0911bd 2024-11-20T19:48:23,101 DEBUG [RS:1;7200dd4cf518:36943 {}] bucket.BucketCache(1487): Thread RS:1;7200dd4cf518:36943 started persisting bucket cache to file 2024-11-20T19:48:23,101 DEBUG [RS:1;7200dd4cf518:36943 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T19:48:23,101 DEBUG [RS:1;7200dd4cf518:36943 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-20T19:48:23,106 DEBUG [RS:1;7200dd4cf518:36943 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-20T19:48:23,106 DEBUG [RS:1;7200dd4cf518:36943 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732132103101 2024-11-20T19:48:23,106 DEBUG [RS:1;7200dd4cf518:36943 {}] bucket.BucketCache(1509): Thread RS:1;7200dd4cf518:36943 finished persisting bucket cache to file, renaming 2024-11-20T19:48:23,106 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T19:48:23,106 INFO [RS:1;7200dd4cf518:36943 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T19:48:23,106 INFO [RS:1;7200dd4cf518:36943 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T19:48:23,106 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T19:48:23,107 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(3091): Received CLOSE for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:23,107 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(959): stopping server 7200dd4cf518,36943,1732132100357 2024-11-20T19:48:23,107 INFO [RS:1;7200dd4cf518:36943 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T19:48:23,107 INFO [RS:1;7200dd4cf518:36943 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7200dd4cf518:36943. 2024-11-20T19:48:23,107 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T19:48:23,107 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06aec7224c925f4cd7c46e2504a5df38, disabling compactions & flushes 2024-11-20T19:48:23,107 DEBUG [RS:1;7200dd4cf518:36943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T19:48:23,107 DEBUG [RS:1;7200dd4cf518:36943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:23,107 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:23,107 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:23,107 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. after waiting 0 ms 2024-11-20T19:48:23,107 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T19:48:23,107 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:23,107 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T19:48:23,107 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T19:48:23,108 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T19:48:23,108 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T19:48:23,108 DEBUG [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(1325): Online Regions={06aec7224c925f4cd7c46e2504a5df38=testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T19:48:23,108 DEBUG [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(1351): Waiting on 06aec7224c925f4cd7c46e2504a5df38, 1588230740 2024-11-20T19:48:23,108 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T19:48:23,108 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T19:48:23,108 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T19:48:23,108 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T19:48:23,109 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 1 ms 2024-11-20T19:48:23,109 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T19:48:23,109 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-20T19:48:23,115 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T19:48:23,115 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:23,116 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06aec7224c925f4cd7c46e2504a5df38: Waiting for close lock at 1732132103107Running coprocessor pre-close hooks at 1732132103107Disabling compacts and flushes for region at 1732132103107Disabling writes for close at 1732132103107Writing region close event to WAL at 1732132103109 (+2 ms)Running coprocessor post-close hooks at 1732132103115 (+6 ms)Closed at 1732132103115 2024-11-20T19:48:23,116 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:23,127 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a6268bc6cecd468aba0f7106def8b7d5 2024-11-20T19:48:23,133 INFO [regionserver/7200dd4cf518:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T19:48:23,141 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/info/a6268bc6cecd468aba0f7106def8b7d5 is 209, key is testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38./info:regioninfo/1732132101811/Put/seqid=0 2024-11-20T19:48:23,142 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/info/a6268bc6cecd468aba0f7106def8b7d5 2024-11-20T19:48:23,152 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 95cffb5a555a41f0af22a2aac6b4d9b0 2024-11-20T19:48:23,166 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/ns/95cffb5a555a41f0af22a2aac6b4d9b0 is 43, key is default/ns:d/1732132101301/Put/seqid=0 2024-11-20T19:48:23,167 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/ns/95cffb5a555a41f0af22a2aac6b4d9b0 2024-11-20T19:48:23,177 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ec645c82ceea40319a6eaf68db623e2d 2024-11-20T19:48:23,197 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/table/ec645c82ceea40319a6eaf68db623e2d is 80, key is testBlockEvictionOnGracefulStop1732132101397/table:state/1732132101824/Put/seqid=0 2024-11-20T19:48:23,198 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/table/ec645c82ceea40319a6eaf68db623e2d 2024-11-20T19:48:23,201 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/info/a6268bc6cecd468aba0f7106def8b7d5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/a6268bc6cecd468aba0f7106def8b7d5 2024-11-20T19:48:23,203 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/a6268bc6cecd468aba0f7106def8b7d5, delay=957 ms 2024-11-20T19:48:23,205 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/a6268bc6cecd468aba0f7106def8b7d5, entries=10, sequenceid=11, filesize=7.0 K 2024-11-20T19:48:23,205 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/ns/95cffb5a555a41f0af22a2aac6b4d9b0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/95cffb5a555a41f0af22a2aac6b4d9b0 2024-11-20T19:48:23,207 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/95cffb5a555a41f0af22a2aac6b4d9b0, delay=917 ms 2024-11-20T19:48:23,208 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/95cffb5a555a41f0af22a2aac6b4d9b0, entries=2, sequenceid=11, filesize=5.0 K 2024-11-20T19:48:23,209 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/table/ec645c82ceea40319a6eaf68db623e2d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/ec645c82ceea40319a6eaf68db623e2d 2024-11-20T19:48:23,210 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/ec645c82ceea40319a6eaf68db623e2d, delay=977 ms 2024-11-20T19:48:23,212 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/ec645c82ceea40319a6eaf68db623e2d, entries=2, sequenceid=11, filesize=5.3 K 2024-11-20T19:48:23,213 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false 2024-11-20T19:48:23,213 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/a6268bc6cecd468aba0f7106def8b7d5 2024-11-20T19:48:23,214 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/95cffb5a555a41f0af22a2aac6b4d9b0 2024-11-20T19:48:23,215 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/ec645c82ceea40319a6eaf68db623e2d 2024-11-20T19:48:23,219 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-20T19:48:23,220 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T19:48:23,220 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T19:48:23,220 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732132103108Running coprocessor pre-close hooks at 1732132103108Disabling compacts and flushes for region at 1732132103108Disabling writes for close at 1732132103109 (+1 ms)Obtaining lock to block concurrent updates at 1732132103109Preparing flush snapshotting stores in 1588230740 at 1732132103109Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732132103111 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732132103111Flushing 1588230740/info: creating writer at 1732132103111Flushing 1588230740/info: appending metadata at 1732132103140 (+29 ms)Flushing 1588230740/info: closing flushed file at 1732132103140Flushing 1588230740/ns: creating writer at 1732132103145 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732132103166 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732132103166Flushing 1588230740/table: creating writer at 1732132103169 (+3 ms)Flushing 1588230740/table: appending metadata at 1732132103196 (+27 ms)Flushing 1588230740/table: closing flushed file at 1732132103196Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61ceae43: reopening flushed file at 1732132103201 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@264a8d3f: reopening flushed file at 1732132103205 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eaddd66: reopening flushed file at 1732132103208 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false at 1732132103213 (+5 ms)Writing region close event to WAL at 1732132103215 (+2 ms)Running coprocessor post-close hooks at 1732132103220 (+5 ms)Closed at 1732132103220 2024-11-20T19:48:23,220 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T19:48:23,308 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(976): stopping server 7200dd4cf518,36943,1732132100357; all regions closed. 2024-11-20T19:48:23,311 DEBUG [RS:1;7200dd4cf518:36943 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs 2024-11-20T19:48:23,311 INFO [RS:1;7200dd4cf518:36943 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7200dd4cf518%2C36943%2C1732132100357.meta:.meta(num 1732132101249) 2024-11-20T19:48:23,314 DEBUG [RS:1;7200dd4cf518:36943 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs 2024-11-20T19:48:23,314 INFO [RS:1;7200dd4cf518:36943 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7200dd4cf518%2C36943%2C1732132100357:(num 1732132100864) 2024-11-20T19:48:23,314 DEBUG [RS:1;7200dd4cf518:36943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:23,314 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T19:48:23,314 INFO [RS:1;7200dd4cf518:36943 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T19:48:23,314 INFO [RS:1;7200dd4cf518:36943 {}] hbase.ChoreService(370): Chore service for: regionserver/7200dd4cf518:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T19:48:23,314 INFO [RS:1;7200dd4cf518:36943 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T19:48:23,314 INFO [regionserver/7200dd4cf518:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T19:48:23,315 INFO [RS:1;7200dd4cf518:36943 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36943 2024-11-20T19:48:23,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7200dd4cf518,36943,1732132100357 2024-11-20T19:48:23,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T19:48:23,319 INFO [RS:1;7200dd4cf518:36943 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T19:48:23,320 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7200dd4cf518,36943,1732132100357] 2024-11-20T19:48:23,327 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7200dd4cf518,36943,1732132100357 already deleted, retry=false 2024-11-20T19:48:23,327 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 7200dd4cf518,36943,1732132100357 on 7200dd4cf518,34433,1732132100273 2024-11-20T19:48:23,332 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 7200dd4cf518,36943,1732132100357, splitWal=true, meta=true 2024-11-20T19:48:23,334 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 7200dd4cf518,36943,1732132100357, splitWal=true, meta=true 2024-11-20T19:48:23,334 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 7200dd4cf518,36943,1732132100357 (carryingMeta=true) 7200dd4cf518,36943,1732132100357/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2f2c424f[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T19:48:23,335 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 7200dd4cf518,36943,1732132100357, splitWal=true, meta=true, isMeta: true 2024-11-20T19:48:23,336 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36943,1732132100357-splitting 2024-11-20T19:48:23,336 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36943,1732132100357-splitting dir is empty, no logs to split. 2024-11-20T19:48:23,336 INFO [PEWorker-2 {}] master.SplitWALManager(105): 7200dd4cf518,36943,1732132100357 WAL count=0, meta=true 2024-11-20T19:48:23,338 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36943,1732132100357-splitting dir is empty, no logs to split. 2024-11-20T19:48:23,338 INFO [PEWorker-2 {}] master.SplitWALManager(105): 7200dd4cf518,36943,1732132100357 WAL count=0, meta=true 2024-11-20T19:48:23,338 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 7200dd4cf518,36943,1732132100357 WAL splitting is done? wals=0, meta=true 2024-11-20T19:48:23,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T19:48:23,340 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T19:48:23,341 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T19:48:23,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:23,426 INFO [RS:1;7200dd4cf518:36943 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T19:48:23,427 INFO [RS:1;7200dd4cf518:36943 {}] regionserver.HRegionServer(1031): Exiting; stopping=7200dd4cf518,36943,1732132100357; zookeeper connection closed. 2024-11-20T19:48:23,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36943-0x10136ec43310002, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:23,427 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2094e5ae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2094e5ae 2024-11-20T19:48:23,491 DEBUG [7200dd4cf518:34433 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T19:48:23,492 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7200dd4cf518,37581,1732132100323 2024-11-20T19:48:23,493 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7200dd4cf518,37581,1732132100323, state=OPENING 2024-11-20T19:48:23,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:23,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:23,495 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:23,495 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T19:48:23,495 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:23,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7200dd4cf518,37581,1732132100323}] 2024-11-20T19:48:23,609 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7200dd4cf518:0 server-side Connection retries=45 2024-11-20T19:48:23,609 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:23,609 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:23,609 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T19:48:23,609 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T19:48:23,609 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T19:48:23,609 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T19:48:23,609 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T19:48:23,610 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37461 2024-11-20T19:48:23,612 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37461 connecting to ZooKeeper ensemble=127.0.0.1:63373 2024-11-20T19:48:23,612 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:23,612 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:23,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374610x0, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T19:48:23,616 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-20T19:48:23,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37461-0x10136ec43310003 connected 2024-11-20T19:48:23,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-20T19:48:23,621 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T19:48:23,621 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.cache 2024-11-20T19:48:23,622 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T19:48:23,622 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T19:48:23,622 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T19:48:23,624 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T19:48:23,624 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T19:48:23,625 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T19:48:23,625 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-20T19:48:23,625 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T19:48:23,626 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-20T19:48:23,627 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T19:48:23,627 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37461 2024-11-20T19:48:23,628 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37461 2024-11-20T19:48:23,628 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37461 2024-11-20T19:48:23,628 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37461 2024-11-20T19:48:23,628 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37461 2024-11-20T19:48:23,630 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(746): ClusterId : 90a8ff9e-87a4-4858-9977-8644c0a34876 2024-11-20T19:48:23,630 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-20T19:48:23,630 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T19:48:23,630 DEBUG [Thread-207 {}] bucket.BucketCache(1584): Cached File Entry:> 2024-11-20T19:48:23,631 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T19:48:23,631 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-20T19:48:23,631 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.persistence is complete. 2024-11-20T19:48:23,633 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T19:48:23,633 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T19:48:23,635 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T19:48:23,635 DEBUG [RS:2;7200dd4cf518:37461 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ec06cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7200dd4cf518/172.17.0.2:0 2024-11-20T19:48:23,648 DEBUG [RS:2;7200dd4cf518:37461 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;7200dd4cf518:37461 2024-11-20T19:48:23,648 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T19:48:23,648 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T19:48:23,648 DEBUG [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T19:48:23,648 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T19:48:23,649 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(2659): reportForDuty to master=7200dd4cf518,34433,1732132100273 with port=37461, startcode=1732132103608 2024-11-20T19:48:23,649 DEBUG [RS:2;7200dd4cf518:37461 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T19:48:23,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60217, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T19:48:23,651 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45919, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T19:48:23,651 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7200dd4cf518,37461,1732132103608 2024-11-20T19:48:23,652 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34433 {}] master.ServerManager(517): Registering regionserver=7200dd4cf518,37461,1732132103608 2024-11-20T19:48:23,653 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T19:48:23,653 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T19:48:23,653 DEBUG [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927 2024-11-20T19:48:23,653 DEBUG [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T19:48:23,653 DEBUG [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T19:48:23,653 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T19:48:23,654 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7200dd4cf518%2C37581%2C1732132100323.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,37581,1732132100323, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs, maxLogs=55 2024-11-20T19:48:23,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T19:48:23,658 DEBUG [RS:2;7200dd4cf518:37461 {}] zookeeper.ZKUtil(111): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7200dd4cf518,37461,1732132103608 2024-11-20T19:48:23,658 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7200dd4cf518,37461,1732132103608] 2024-11-20T19:48:23,658 WARN [RS:2;7200dd4cf518:37461 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T19:48:23,658 INFO [RS:2;7200dd4cf518:37461 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T19:48:23,658 DEBUG [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,37461,1732132103608 2024-11-20T19:48:23,663 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T19:48:23,666 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T19:48:23,667 INFO [RS:2;7200dd4cf518:37461 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T19:48:23,667 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:23,668 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T19:48:23,668 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T19:48:23,668 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:23,668 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,668 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7200dd4cf518:0, corePoolSize=2, maxPoolSize=2 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7200dd4cf518:0, corePoolSize=1, maxPoolSize=1 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7200dd4cf518:0, corePoolSize=3, maxPoolSize=3 2024-11-20T19:48:23,669 DEBUG [RS:2;7200dd4cf518:37461 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7200dd4cf518:0, corePoolSize=3, maxPoolSize=3 2024-11-20T19:48:23,672 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:23,672 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:23,672 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:23,672 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:23,672 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:23,672 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,37461,1732132103608-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T19:48:23,681 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,37581,1732132100323/7200dd4cf518%2C37581%2C1732132100323.meta.1732132103655.meta 2024-11-20T19:48:23,681 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T19:48:23,682 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T19:48:23,682 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T19:48:23,682 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T19:48:23,682 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T19:48:23,682 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T19:48:23,682 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:23,682 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T19:48:23,682 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T19:48:23,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T19:48:23,684 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T19:48:23,684 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:23,686 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/a6268bc6cecd468aba0f7106def8b7d5, delay=963 ms 2024-11-20T19:48:23,687 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T19:48:23,687 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=7200dd4cf518,37461,1732132103608-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:23,687 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:23,687 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.Replication(171): 7200dd4cf518,37461,1732132103608 started 2024-11-20T19:48:23,688 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/a6268bc6cecd468aba0f7106def8b7d5 2024-11-20T19:48:23,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:23,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T19:48:23,689 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T19:48:23,689 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:23,690 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/95cffb5a555a41f0af22a2aac6b4d9b0, delay=955 ms 2024-11-20T19:48:23,691 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/95cffb5a555a41f0af22a2aac6b4d9b0 2024-11-20T19:48:23,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:23,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T19:48:23,692 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T19:48:23,692 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:23,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:23,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T19:48:23,693 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T19:48:23,693 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:23,694 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/ec645c82ceea40319a6eaf68db623e2d, delay=901 ms 2024-11-20T19:48:23,695 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/ec645c82ceea40319a6eaf68db623e2d 2024-11-20T19:48:23,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T19:48:23,695 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T19:48:23,696 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740 2024-11-20T19:48:23,696 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740 2024-11-20T19:48:23,697 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T19:48:23,697 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T19:48:23,697 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T19:48:23,698 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T19:48:23,698 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61528120, jitterRate=-0.08315956592559814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T19:48:23,698 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T19:48:23,699 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732132103683Writing region info on filesystem at 1732132103683Initializing all the Stores at 1732132103683Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132103683Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132103683Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132103683Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732132103683Cleaning up temporary data from old regions at 1732132103697 (+14 ms)Running coprocessor post-open hooks at 1732132103698 (+1 ms)Region opened successfully at 1732132103699 (+1 ms) 2024-11-20T19:48:23,700 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732132103648 2024-11-20T19:48:23,703 DEBUG [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T19:48:23,703 INFO [RS_OPEN_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T19:48:23,704 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=7200dd4cf518,37581,1732132100323 2024-11-20T19:48:23,705 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7200dd4cf518,37581,1732132100323, state=OPEN 2024-11-20T19:48:23,705 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T19:48:23,705 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(1482): Serving as 7200dd4cf518,37461,1732132103608, RpcServer on 7200dd4cf518/172.17.0.2:37461, sessionid=0x10136ec43310003 2024-11-20T19:48:23,706 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T19:48:23,706 DEBUG [RS:2;7200dd4cf518:37461 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7200dd4cf518,37461,1732132103608 2024-11-20T19:48:23,706 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7200dd4cf518,37461,1732132103608' 2024-11-20T19:48:23,706 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T19:48:23,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:23,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:23,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T19:48:23,707 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7200dd4cf518,37581,1732132100323 2024-11-20T19:48:23,707 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T19:48:23,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:23,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:23,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T19:48:23,707 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T19:48:23,707 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T19:48:23,708 DEBUG [RS:2;7200dd4cf518:37461 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7200dd4cf518,37461,1732132103608 2024-11-20T19:48:23,708 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7200dd4cf518,37461,1732132103608' 2024-11-20T19:48:23,708 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T19:48:23,708 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T19:48:23,709 DEBUG [RS:2;7200dd4cf518:37461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T19:48:23,709 INFO [RS:2;7200dd4cf518:37461 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T19:48:23,709 INFO [RS:2;7200dd4cf518:37461 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T19:48:23,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-20T19:48:23,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7200dd4cf518,37581,1732132100323 in 212 msec 2024-11-20T19:48:23,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T19:48:23,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 371 msec 2024-11-20T19:48:23,712 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(207): 7200dd4cf518,36943,1732132100357 had 2 regions 2024-11-20T19:48:23,713 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 7200dd4cf518,36943,1732132100357, splitWal=true, meta=true, isMeta: false 2024-11-20T19:48:23,714 INFO [PEWorker-4 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36943,1732132100357-splitting dir is empty, no logs to split. 2024-11-20T19:48:23,714 INFO [PEWorker-4 {}] master.SplitWALManager(105): 7200dd4cf518,36943,1732132100357 WAL count=0, meta=false 2024-11-20T19:48:23,714 INFO [PEWorker-4 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,36943,1732132100357-splitting dir is empty, no logs to split. 2024-11-20T19:48:23,715 INFO [PEWorker-4 {}] master.SplitWALManager(105): 7200dd4cf518,36943,1732132100357 WAL count=0, meta=false 2024-11-20T19:48:23,715 DEBUG [PEWorker-4 {}] procedure.ServerCrashProcedure(329): Check if 7200dd4cf518,36943,1732132100357 WAL splitting is done? wals=0, meta=false 2024-11-20T19:48:23,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732132101397, region=06aec7224c925f4cd7c46e2504a5df38, ASSIGN}] 2024-11-20T19:48:23,717 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732132101397, region=06aec7224c925f4cd7c46e2504a5df38, ASSIGN 2024-11-20T19:48:23,718 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732132101397, region=06aec7224c925f4cd7c46e2504a5df38, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T19:48:23,810 INFO [RS:2;7200dd4cf518:37461 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T19:48:23,811 INFO [RS:2;7200dd4cf518:37461 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7200dd4cf518%2C37461%2C1732132103608, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,37461,1732132103608, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs, maxLogs=55 2024-11-20T19:48:23,833 INFO [RS:2;7200dd4cf518:37461 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/WALs/7200dd4cf518,37461,1732132103608/7200dd4cf518%2C37461%2C1732132103608.1732132103811 2024-11-20T19:48:23,833 DEBUG [RS:2;7200dd4cf518:37461 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T19:48:23,868 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(204): Hosts are {7200dd4cf518=0} racks are {/default-rack=0} 2024-11-20T19:48:23,869 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T19:48:23,869 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T19:48:23,869 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T19:48:23,869 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T19:48:23,869 INFO [7200dd4cf518:34433 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T19:48:23,869 INFO [7200dd4cf518:34433 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T19:48:23,869 DEBUG [7200dd4cf518:34433 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T19:48:23,869 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06aec7224c925f4cd7c46e2504a5df38, regionState=OPENING, regionLocation=7200dd4cf518,37581,1732132100323 2024-11-20T19:48:23,871 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 7200dd4cf518:36943 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7200dd4cf518/172.17.0.2:36943 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T19:48:23,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7200dd4cf518,36943,1732132100357, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=7200dd4cf518,36943,1732132100357, seqNum=-1, error=java.net.ConnectException: Call to address=7200dd4cf518:36943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7200dd4cf518/172.17.0.2:36943 2024-11-20T19:48:23,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=7200dd4cf518,36943,1732132100357, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T19:48:23,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=7200dd4cf518,36943,1732132100357, seqNum=-1 from cache 2024-11-20T19:48:23,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 7200dd4cf518:36943 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7200dd4cf518/172.17.0.2:36943 2024-11-20T19:48:23,978 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T19:48:23,979 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7200dd4cf518,37581,1732132100323, seqNum=-1] 2024-11-20T19:48:23,979 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T19:48:23,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56005, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T19:48:23,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732132101397, region=06aec7224c925f4cd7c46e2504a5df38, ASSIGN because future has completed 2024-11-20T19:48:23,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06aec7224c925f4cd7c46e2504a5df38, server=7200dd4cf518,37581,1732132100323}] 2024-11-20T19:48:24,140 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:24,140 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 06aec7224c925f4cd7c46e2504a5df38, NAME => 'testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38.', STARTKEY => '', ENDKEY => ''} 2024-11-20T19:48:24,141 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732132101397 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,141 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T19:48:24,141 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,141 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,142 INFO [StoreOpener-06aec7224c925f4cd7c46e2504a5df38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,143 INFO [StoreOpener-06aec7224c925f4cd7c46e2504a5df38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06aec7224c925f4cd7c46e2504a5df38 columnFamilyName family 2024-11-20T19:48:24,144 DEBUG [StoreOpener-06aec7224c925f4cd7c46e2504a5df38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T19:48:24,149 DEBUG [StoreFileOpener-06aec7224c925f4cd7c46e2504a5df38-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd, delay=919 ms 2024-11-20T19:48:24,151 DEBUG [StoreOpener-06aec7224c925f4cd7c46e2504a5df38-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd 2024-11-20T19:48:24,151 INFO [StoreOpener-06aec7224c925f4cd7c46e2504a5df38-1 {}] regionserver.HStore(327): Store=06aec7224c925f4cd7c46e2504a5df38/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T19:48:24,151 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,152 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,153 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,153 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,153 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,155 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,155 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 06aec7224c925f4cd7c46e2504a5df38; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73037429, jitterRate=0.08834250271320343}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T19:48:24,155 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,157 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 06aec7224c925f4cd7c46e2504a5df38: Running coprocessor pre-open hook at 1732132104142Writing region info on filesystem at 1732132104142Initializing all the Stores at 1732132104142Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732132104142Cleaning up temporary data from old regions at 1732132104153 (+11 ms)Running coprocessor post-open hooks at 1732132104155 (+2 ms)Region opened successfully at 1732132104156 (+1 ms) 2024-11-20T19:48:24,157 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38., pid=11, masterSystemTime=1732132104137 2024-11-20T19:48:24,161 DEBUG [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:24,161 INFO [RS_OPEN_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:24,161 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06aec7224c925f4cd7c46e2504a5df38, regionState=OPEN, openSeqNum=10, regionLocation=7200dd4cf518,37581,1732132100323 2024-11-20T19:48:24,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06aec7224c925f4cd7c46e2504a5df38, server=7200dd4cf518,37581,1732132100323 because future has completed 2024-11-20T19:48:24,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T19:48:24,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 06aec7224c925f4cd7c46e2504a5df38, server=7200dd4cf518,37581,1732132100323 in 180 msec 2024-11-20T19:48:24,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-20T19:48:24,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732132101397, region=06aec7224c925f4cd7c46e2504a5df38, ASSIGN in 451 msec 2024-11-20T19:48:24,169 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 7200dd4cf518,36943,1732132100357 after splitting done 2024-11-20T19:48:24,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 7200dd4cf518,36943,1732132100357, splitWal=true, meta=true in 841 msec 2024-11-20T19:48:24,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T19:48:24,206 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T19:48:24,206 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T19:48:24,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:24,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:24,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T19:48:24,207 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T19:48:24,207 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=661207564, stopped=false 2024-11-20T19:48:24,207 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7200dd4cf518,34433,1732132100273 2024-11-20T19:48:24,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:24,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:24,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:24,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T19:48:24,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:24,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T19:48:24,210 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T19:48:24,210 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T19:48:24,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:24,210 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7200dd4cf518,37581,1732132100323' ***** 2024-11-20T19:48:24,211 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T19:48:24,211 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7200dd4cf518,37461,1732132103608' ***** 2024-11-20T19:48:24,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:24,211 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T19:48:24,211 INFO [RS:0;7200dd4cf518:37581 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T19:48:24,211 INFO [RS:0;7200dd4cf518:37581 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.persistence 2024-11-20T19:48:24,211 INFO [RS:2;7200dd4cf518:37461 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T19:48:24,212 INFO [RS:2;7200dd4cf518:37461 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3f81ac6b-b24e-97e2-bf81-1d57d96aef20/bucket.persistence 2024-11-20T19:48:24,212 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T19:48:24,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:24,212 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T19:48:24,212 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T19:48:24,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:24,211 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T19:48:24,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T19:48:24,212 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T19:48:24,212 INFO [RS:0;7200dd4cf518:37581 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T19:48:24,213 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T19:48:24,215 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T19:48:24,215 INFO [RS:2;7200dd4cf518:37461 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T19:48:24,215 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T19:48:24,223 DEBUG [RS:0;7200dd4cf518:37581 {}] bucket.BucketCache(1487): Thread RS:0;7200dd4cf518:37581 started persisting bucket cache to file 2024-11-20T19:48:24,223 DEBUG [RS:0;7200dd4cf518:37581 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T19:48:24,223 DEBUG [RS:0;7200dd4cf518:37581 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T19:48:24,225 DEBUG [RS:2;7200dd4cf518:37461 {}] bucket.BucketCache(1487): Thread RS:2;7200dd4cf518:37461 started persisting bucket cache to file 2024-11-20T19:48:24,225 DEBUG [RS:2;7200dd4cf518:37461 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T19:48:24,225 DEBUG [RS:2;7200dd4cf518:37461 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-20T19:48:24,227 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T19:48:24,227 INFO [regionserver/7200dd4cf518:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T19:48:24,227 DEBUG [RS:0;7200dd4cf518:37581 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T19:48:24,227 DEBUG [RS:0;7200dd4cf518:37581 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732132104223 2024-11-20T19:48:24,227 DEBUG [RS:0;7200dd4cf518:37581 {}] bucket.BucketCache(1509): Thread RS:0;7200dd4cf518:37581 finished persisting bucket cache to file, renaming 2024-11-20T19:48:24,228 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T19:48:24,228 INFO [RS:0;7200dd4cf518:37581 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T19:48:24,228 INFO [RS:0;7200dd4cf518:37581 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T19:48:24,229 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(3091): Received CLOSE for 06aec7224c925f4cd7c46e2504a5df38 2024-11-20T19:48:24,229 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(959): stopping server 7200dd4cf518,37581,1732132100323 2024-11-20T19:48:24,229 INFO [RS:0;7200dd4cf518:37581 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T19:48:24,229 DEBUG [RS:2;7200dd4cf518:37461 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-20T19:48:24,229 INFO [RS:0;7200dd4cf518:37581 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7200dd4cf518:37581. 2024-11-20T19:48:24,229 DEBUG [RS:2;7200dd4cf518:37461 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732132104225 2024-11-20T19:48:24,229 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T19:48:24,229 DEBUG [RS:2;7200dd4cf518:37461 {}] bucket.BucketCache(1509): Thread RS:2;7200dd4cf518:37461 finished persisting bucket cache to file, renaming 2024-11-20T19:48:24,229 DEBUG [RS:0;7200dd4cf518:37581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T19:48:24,229 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06aec7224c925f4cd7c46e2504a5df38, disabling compactions & flushes 2024-11-20T19:48:24,229 DEBUG [RS:0;7200dd4cf518:37581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:24,229 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:24,229 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T19:48:24,229 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T19:48:24,229 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:24,230 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T19:48:24,230 INFO [RS:2;7200dd4cf518:37461 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T19:48:24,230 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T19:48:24,230 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T19:48:24,230 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. after waiting 0 ms 2024-11-20T19:48:24,230 INFO [RS:2;7200dd4cf518:37461 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T19:48:24,230 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T19:48:24,230 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:24,230 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(959): stopping server 7200dd4cf518,37461,1732132103608 2024-11-20T19:48:24,230 INFO [RS:2;7200dd4cf518:37461 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T19:48:24,230 INFO [RS:2;7200dd4cf518:37461 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;7200dd4cf518:37461. 2024-11-20T19:48:24,230 DEBUG [RS:2;7200dd4cf518:37461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T19:48:24,230 DEBUG [RS:2;7200dd4cf518:37461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:24,231 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(976): stopping server 7200dd4cf518,37461,1732132103608; all regions closed. 2024-11-20T19:48:24,232 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T19:48:24,232 DEBUG [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(1325): Online Regions={06aec7224c925f4cd7c46e2504a5df38=testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T19:48:24,232 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T19:48:24,232 DEBUG [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(1351): Waiting on 06aec7224c925f4cd7c46e2504a5df38, 1588230740 2024-11-20T19:48:24,232 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T19:48:24,232 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T19:48:24,232 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T19:48:24,232 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T19:48:24,232 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T19:48:24,232 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-20T19:48:24,236 DEBUG [StoreFileCloser-06aec7224c925f4cd7c46e2504a5df38-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/family/a809000669484d2a8a16589a6b0911bd 2024-11-20T19:48:24,243 DEBUG [RS:2;7200dd4cf518:37461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs 2024-11-20T19:48:24,243 INFO [RS:2;7200dd4cf518:37461 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7200dd4cf518%2C37461%2C1732132103608:(num 1732132103811) 2024-11-20T19:48:24,243 DEBUG [RS:2;7200dd4cf518:37461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:24,243 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T19:48:24,243 INFO [RS:2;7200dd4cf518:37461 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T19:48:24,243 INFO [RS:2;7200dd4cf518:37461 {}] hbase.ChoreService(370): Chore service for: regionserver/7200dd4cf518:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T19:48:24,243 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T19:48:24,243 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T19:48:24,243 INFO [regionserver/7200dd4cf518:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T19:48:24,243 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T19:48:24,243 INFO [RS:2;7200dd4cf518:37461 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T19:48:24,243 INFO [RS:2;7200dd4cf518:37461 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37461 2024-11-20T19:48:24,245 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/default/testBlockEvictionOnGracefulStop1732132101397/06aec7224c925f4cd7c46e2504a5df38/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T19:48:24,245 INFO [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:24,246 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06aec7224c925f4cd7c46e2504a5df38: Waiting for close lock at 1732132104229Running coprocessor pre-close hooks at 1732132104229Disabling compacts and flushes for region at 1732132104229Disabling writes for close at 1732132104230 (+1 ms)Writing region close event to WAL at 1732132104237 (+7 ms)Running coprocessor post-close hooks at 1732132104245 (+8 ms)Closed at 1732132104245 2024-11-20T19:48:24,246 DEBUG [RS_CLOSE_REGION-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38. 2024-11-20T19:48:24,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T19:48:24,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7200dd4cf518,37461,1732132103608 2024-11-20T19:48:24,246 INFO [RS:2;7200dd4cf518:37461 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T19:48:24,249 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7200dd4cf518,37461,1732132103608] 2024-11-20T19:48:24,252 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7200dd4cf518,37461,1732132103608 already deleted, retry=false 2024-11-20T19:48:24,252 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7200dd4cf518,37461,1732132103608 expired; onlineServers=1 2024-11-20T19:48:24,258 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c8cc7356394648b592dbc96bfee9b4e0 2024-11-20T19:48:24,273 INFO [regionserver/7200dd4cf518:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T19:48:24,282 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/info/c8cc7356394648b592dbc96bfee9b4e0 is 209, key is testBlockEvictionOnGracefulStop1732132101397,,1732132101402.06aec7224c925f4cd7c46e2504a5df38./info:regioninfo/1732132104161/Put/seqid=0 2024-11-20T19:48:24,283 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/info/c8cc7356394648b592dbc96bfee9b4e0 2024-11-20T19:48:24,286 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/.tmp/info/c8cc7356394648b592dbc96bfee9b4e0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/c8cc7356394648b592dbc96bfee9b4e0 2024-11-20T19:48:24,287 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/c8cc7356394648b592dbc96bfee9b4e0, delay=902 ms 2024-11-20T19:48:24,288 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/c8cc7356394648b592dbc96bfee9b4e0, entries=8, sequenceid=19, filesize=6.6 K 2024-11-20T19:48:24,289 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 57ms, sequenceid=19, compaction requested=false 2024-11-20T19:48:24,290 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/a6268bc6cecd468aba0f7106def8b7d5 2024-11-20T19:48:24,290 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/info/c8cc7356394648b592dbc96bfee9b4e0 2024-11-20T19:48:24,291 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/ns/95cffb5a555a41f0af22a2aac6b4d9b0 2024-11-20T19:48:24,291 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/table/ec645c82ceea40319a6eaf68db623e2d 2024-11-20T19:48:24,297 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-20T19:48:24,297 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T19:48:24,297 INFO [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T19:48:24,297 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732132104232Running coprocessor pre-close hooks at 1732132104232Disabling compacts and flushes for region at 1732132104232Disabling writes for close at 1732132104232Obtaining lock to block concurrent updates at 1732132104232Preparing flush snapshotting stores in 1588230740 at 1732132104232Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732132104233 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732132104233Flushing 1588230740/info: creating writer at 1732132104233Flushing 1588230740/info: appending metadata at 1732132104281 (+48 ms)Flushing 1588230740/info: closing flushed file at 1732132104281Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b3bf55b: reopening flushed file at 1732132104285 (+4 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 57ms, sequenceid=19, compaction requested=false at 1732132104289 (+4 ms)Writing region close event to WAL at 1732132104292 (+3 ms)Running coprocessor post-close hooks at 1732132104297 (+5 ms)Closed at 1732132104297 2024-11-20T19:48:24,297 DEBUG [RS_CLOSE_META-regionserver/7200dd4cf518:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T19:48:24,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:24,349 INFO [RS:2;7200dd4cf518:37461 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T19:48:24,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37461-0x10136ec43310003, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:24,349 INFO [RS:2;7200dd4cf518:37461 {}] regionserver.HRegionServer(1031): Exiting; stopping=7200dd4cf518,37461,1732132103608; zookeeper connection closed. 2024-11-20T19:48:24,349 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@17c1c035 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@17c1c035 2024-11-20T19:48:24,432 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(976): stopping server 7200dd4cf518,37581,1732132100323; all regions closed. 2024-11-20T19:48:24,435 DEBUG [RS:0;7200dd4cf518:37581 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs 2024-11-20T19:48:24,435 INFO [RS:0;7200dd4cf518:37581 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7200dd4cf518%2C37581%2C1732132100323.meta:.meta(num 1732132103655) 2024-11-20T19:48:24,437 DEBUG [RS:0;7200dd4cf518:37581 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/oldWALs 2024-11-20T19:48:24,438 INFO [RS:0;7200dd4cf518:37581 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7200dd4cf518%2C37581%2C1732132100323:(num 1732132100861) 2024-11-20T19:48:24,438 DEBUG [RS:0;7200dd4cf518:37581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T19:48:24,438 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T19:48:24,438 INFO [RS:0;7200dd4cf518:37581 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T19:48:24,438 INFO [RS:0;7200dd4cf518:37581 {}] hbase.ChoreService(370): Chore service for: regionserver/7200dd4cf518:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T19:48:24,438 INFO [RS:0;7200dd4cf518:37581 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T19:48:24,438 INFO [regionserver/7200dd4cf518:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T19:48:24,439 INFO [RS:0;7200dd4cf518:37581 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37581 2024-11-20T19:48:24,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7200dd4cf518,37581,1732132100323 2024-11-20T19:48:24,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T19:48:24,442 INFO [RS:0;7200dd4cf518:37581 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T19:48:24,444 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7200dd4cf518,37581,1732132100323] 2024-11-20T19:48:24,446 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7200dd4cf518,37581,1732132100323 already deleted, retry=false 2024-11-20T19:48:24,446 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7200dd4cf518,37581,1732132100323 expired; onlineServers=0 2024-11-20T19:48:24,446 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7200dd4cf518,34433,1732132100273' ***** 2024-11-20T19:48:24,446 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T19:48:24,446 INFO [M:0;7200dd4cf518:34433 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T19:48:24,446 INFO [M:0;7200dd4cf518:34433 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T19:48:24,446 DEBUG [M:0;7200dd4cf518:34433 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T19:48:24,446 DEBUG [M:0;7200dd4cf518:34433 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T19:48:24,446 DEBUG [master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.large.0-1732132100627 {}] cleaner.HFileCleaner(306): Exit Thread[master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.large.0-1732132100627,5,FailOnTimeoutGroup] 2024-11-20T19:48:24,446 DEBUG [master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.small.0-1732132100627 {}] cleaner.HFileCleaner(306): Exit Thread[master/7200dd4cf518:0:becomeActiveMaster-HFileCleaner.small.0-1732132100627,5,FailOnTimeoutGroup] 2024-11-20T19:48:24,446 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T19:48:24,447 INFO [M:0;7200dd4cf518:34433 {}] hbase.ChoreService(370): Chore service for: master/7200dd4cf518:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T19:48:24,447 INFO [M:0;7200dd4cf518:34433 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T19:48:24,447 DEBUG [M:0;7200dd4cf518:34433 {}] master.HMaster(1795): Stopping service threads 2024-11-20T19:48:24,447 INFO [M:0;7200dd4cf518:34433 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T19:48:24,447 INFO [M:0;7200dd4cf518:34433 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T19:48:24,447 INFO [M:0;7200dd4cf518:34433 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T19:48:24,447 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T19:48:24,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T19:48:24,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T19:48:24,457 DEBUG [M:0;7200dd4cf518:34433 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-20T19:48:24,457 DEBUG [M:0;7200dd4cf518:34433 {}] master.ActiveMasterManager(353): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-20T19:48:24,457 INFO [M:0;7200dd4cf518:34433 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/.lastflushedseqids 2024-11-20T19:48:24,464 INFO [M:0;7200dd4cf518:34433 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T19:48:24,465 INFO [M:0;7200dd4cf518:34433 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T19:48:24,465 DEBUG [M:0;7200dd4cf518:34433 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T19:48:24,465 INFO [M:0;7200dd4cf518:34433 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:24,465 DEBUG [M:0;7200dd4cf518:34433 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:24,465 DEBUG [M:0;7200dd4cf518:34433 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T19:48:24,465 DEBUG [M:0;7200dd4cf518:34433 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:24,465 INFO [M:0;7200dd4cf518:34433 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-20T19:48:24,477 DEBUG [M:0;7200dd4cf518:34433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9b57e2ee8b8342f4b94d99776398537a 2024-11-20T19:48:24,492 DEBUG [M:0;7200dd4cf518:34433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b57e2ee8b8342f4b94d99776398537a is 82, key is hbase:meta,,1/info:regioninfo/1732132103703/Put/seqid=0 2024-11-20T19:48:24,492 INFO [M:0;7200dd4cf518:34433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b57e2ee8b8342f4b94d99776398537a 2024-11-20T19:48:24,502 DEBUG [M:0;7200dd4cf518:34433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5b20175f5d954fdf8acdc3be2c8a3cc8 2024-11-20T19:48:24,524 DEBUG [M:0;7200dd4cf518:34433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b20175f5d954fdf8acdc3be2c8a3cc8 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732132101830/Put/seqid=0 2024-11-20T19:48:24,525 INFO [M:0;7200dd4cf518:34433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b20175f5d954fdf8acdc3be2c8a3cc8 2024-11-20T19:48:24,535 DEBUG [M:0;7200dd4cf518:34433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 01064a0ef3474dba8ffe6fc5c155c290 2024-11-20T19:48:24,545 INFO [RS:0;7200dd4cf518:37581 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T19:48:24,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:24,545 INFO [RS:0;7200dd4cf518:37581 {}] regionserver.HRegionServer(1031): Exiting; stopping=7200dd4cf518,37581,1732132100323; zookeeper connection closed. 2024-11-20T19:48:24,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37581-0x10136ec43310001, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:24,545 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@458b0e7b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@458b0e7b 2024-11-20T19:48:24,545 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T19:48:24,552 DEBUG [M:0;7200dd4cf518:34433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01064a0ef3474dba8ffe6fc5c155c290 is 69, key is 7200dd4cf518,37461,1732132103608/rs:state/1732132103652/Put/seqid=0 2024-11-20T19:48:24,552 INFO [M:0;7200dd4cf518:34433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01064a0ef3474dba8ffe6fc5c155c290 2024-11-20T19:48:24,554 INFO [M:0;7200dd4cf518:34433 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01064a0ef3474dba8ffe6fc5c155c290 2024-11-20T19:48:24,555 DEBUG [M:0;7200dd4cf518:34433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b57e2ee8b8342f4b94d99776398537a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9b57e2ee8b8342f4b94d99776398537a 2024-11-20T19:48:24,556 INFO [M:0;7200dd4cf518:34433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9b57e2ee8b8342f4b94d99776398537a, entries=14, sequenceid=102, filesize=5.9 K 2024-11-20T19:48:24,557 DEBUG [M:0;7200dd4cf518:34433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b20175f5d954fdf8acdc3be2c8a3cc8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5b20175f5d954fdf8acdc3be2c8a3cc8 2024-11-20T19:48:24,558 INFO [M:0;7200dd4cf518:34433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5b20175f5d954fdf8acdc3be2c8a3cc8, entries=11, sequenceid=102, filesize=6.5 K 2024-11-20T19:48:24,559 DEBUG [M:0;7200dd4cf518:34433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01064a0ef3474dba8ffe6fc5c155c290 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/01064a0ef3474dba8ffe6fc5c155c290 2024-11-20T19:48:24,561 INFO [M:0;7200dd4cf518:34433 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01064a0ef3474dba8ffe6fc5c155c290 2024-11-20T19:48:24,561 INFO [M:0;7200dd4cf518:34433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a142c9c6-d4e3-db08-3709-8e054527c89b/f383dc1e-643c-6d4f-80a0-51a08e078927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/01064a0ef3474dba8ffe6fc5c155c290, entries=3, sequenceid=102, filesize=5.3 K 2024-11-20T19:48:24,562 INFO [M:0;7200dd4cf518:34433 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=102, compaction requested=false 2024-11-20T19:48:24,565 INFO [M:0;7200dd4cf518:34433 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T19:48:24,565 DEBUG [M:0;7200dd4cf518:34433 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732132104465Disabling compacts and flushes for region at 1732132104465Disabling writes for close at 1732132104465Obtaining lock to block concurrent updates at 1732132104465Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732132104465Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732132104466 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732132104466Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732132104466Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732132104491 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732132104491Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732132104495 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732132104523 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732132104523Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732132104527 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732132104551 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732132104551Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47ad038e: reopening flushed file at 1732132104554 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f70cf3c: reopening flushed file at 1732132104556 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c6b18ba: reopening flushed file at 1732132104559 (+3 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=102, compaction requested=false at 1732132104562 (+3 ms)Writing region close event to WAL at 1732132104564 (+2 ms)Closed at 1732132104564 2024-11-20T19:48:24,566 INFO [M:0;7200dd4cf518:34433 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T19:48:24,566 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T19:48:24,566 INFO [M:0;7200dd4cf518:34433 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34433 2024-11-20T19:48:24,567 INFO [M:0;7200dd4cf518:34433 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T19:48:24,670 INFO [M:0;7200dd4cf518:34433 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T19:48:24,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:24,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34433-0x10136ec43310000, quorum=127.0.0.1:63373, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T19:48:24,673 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T19:48:24,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T19:48:24,675 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T19:48:24,684 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=292 (was 257) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5839 (was 5353) - AvailableMemoryMB LEAK? -