2024-11-22 23:44:58,964 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-22 23:44:58,981 main DEBUG Took 0.015294 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 23:44:58,981 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 23:44:58,982 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 23:44:58,982 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 23:44:58,984 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:58,991 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 23:44:59,002 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,003 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,004 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,006 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,006 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,007 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,008 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,008 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,009 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,010 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,010 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,011 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,011 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,011 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,012 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:44:59,013 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,013 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 23:44:59,014 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:44:59,016 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 23:44:59,017 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 23:44:59,018 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 23:44:59,019 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 23:44:59,019 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 23:44:59,028 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 23:44:59,030 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 23:44:59,032 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 23:44:59,032 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 23:44:59,033 main DEBUG createAppenders(={Console}) 2024-11-22 23:44:59,033 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-22 23:44:59,034 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-22 23:44:59,034 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-22 23:44:59,034 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 23:44:59,035 main DEBUG OutputStream closed 2024-11-22 23:44:59,035 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 23:44:59,035 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 23:44:59,035 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-22 23:44:59,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 23:44:59,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 23:44:59,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 23:44:59,110 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 23:44:59,111 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 23:44:59,112 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 23:44:59,112 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 23:44:59,113 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 23:44:59,113 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 23:44:59,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 23:44:59,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 23:44:59,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 23:44:59,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 23:44:59,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 23:44:59,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 23:44:59,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 23:44:59,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 23:44:59,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 23:44:59,120 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 23:44:59,120 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-22 23:44:59,120 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 23:44:59,121 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-22T23:44:59,134 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-22 23:44:59,138 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 23:44:59,138 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T23:44:59,364 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28 2024-11-22T23:44:59,386 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=9634 2024-11-22T23:44:59,493 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T23:44:59,584 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:44:59,599 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/cluster_2daa302e-9894-0dd6-3e6d-7e81e8474fa8, deleteOnExit=true 2024-11-22T23:44:59,697 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/cluster_2daa302e-9894-0dd6-3e6d-7e81e8474fa8/zookeeper_0, clientPort=50231, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/cluster_2daa302e-9894-0dd6-3e6d-7e81e8474fa8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/cluster_2daa302e-9894-0dd6-3e6d-7e81e8474fa8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T23:44:59,710 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50231 2024-11-22T23:44:59,711 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:44:59,797 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 with version=8 2024-11-22T23:44:59,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/hbase-staging 2024-11-22T23:44:59,896 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T23:45:00,171 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:45:00,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:00,184 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:00,195 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:45:00,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:00,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:45:00,333 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T23:45:00,419 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:45:00,457 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T23:45:00,468 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T23:45:00,472 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:45:00,509 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15263 (auto-detected) 2024-11-22T23:45:00,510 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T23:45:00,531 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41027 2024-11-22T23:45:00,555 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41027 connecting to ZooKeeper ensemble=127.0.0.1:50231 2024-11-22T23:45:00,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:410270x0, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:45:00,593 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41027-0x1014211a5a70000 connected 2024-11-22T23:45:00,622 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:00,623 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:00,630 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:00,635 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532, hbase.cluster.distributed=false 2024-11-22T23:45:00,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:45:00,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41027 2024-11-22T23:45:00,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41027 2024-11-22T23:45:00,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41027 2024-11-22T23:45:00,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41027 2024-11-22T23:45:00,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41027 2024-11-22T23:45:00,759 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:45:00,760 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:00,761 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:00,761 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:45:00,761 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:00,761 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:45:00,764 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T23:45:00,767 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:45:00,768 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33031 2024-11-22T23:45:00,769 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33031 connecting to ZooKeeper ensemble=127.0.0.1:50231 2024-11-22T23:45:00,770 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:00,771 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:00,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330310x0, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:45:00,776 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33031-0x1014211a5a70001 connected 2024-11-22T23:45:00,776 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:00,781 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T23:45:00,796 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/bucket.cache 2024-11-22T23:45:00,804 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T23:45:00,805 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T23:45:00,807 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T23:45:00,809 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T23:45:00,809 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T23:45:00,816 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T23:45:00,821 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T23:45:00,821 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/bucket.persistence is complete. 2024-11-22T23:45:00,823 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T23:45:00,826 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T23:45:00,831 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:45:00,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33031 2024-11-22T23:45:00,837 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33031 2024-11-22T23:45:00,838 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33031 2024-11-22T23:45:00,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33031 2024-11-22T23:45:00,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33031 2024-11-22T23:45:00,860 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:45:00,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:00,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:00,860 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:45:00,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:00,861 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:45:00,861 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T23:45:00,861 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:45:00,864 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34619 2024-11-22T23:45:00,866 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34619 connecting to ZooKeeper ensemble=127.0.0.1:50231 2024-11-22T23:45:00,866 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:00,867 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:00,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:346190x0, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:45:00,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:346190x0, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:00,873 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T23:45:00,873 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34619-0x1014211a5a70002 connected 2024-11-22T23:45:00,874 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/bucket.cache 2024-11-22T23:45:00,874 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T23:45:00,874 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T23:45:00,875 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T23:45:00,875 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T23:45:00,875 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T23:45:00,875 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T23:45:00,876 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T23:45:00,876 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T23:45:00,876 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/bucket.persistence is complete. 2024-11-22T23:45:00,878 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T23:45:00,880 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:45:00,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34619 2024-11-22T23:45:00,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34619 2024-11-22T23:45:00,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34619 2024-11-22T23:45:00,889 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34619 2024-11-22T23:45:00,889 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34619 2024-11-22T23:45:00,906 DEBUG [M:0;a511e45b3024:41027 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a511e45b3024:41027 2024-11-22T23:45:00,907 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a511e45b3024,41027,1732319099972 2024-11-22T23:45:00,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:00,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:00,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:00,920 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a511e45b3024,41027,1732319099972 2024-11-22T23:45:00,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T23:45:00,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T23:45:00,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:00,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:00,948 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T23:45:00,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:00,949 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a511e45b3024,41027,1732319099972 from backup master directory 2024-11-22T23:45:00,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a511e45b3024,41027,1732319099972 2024-11-22T23:45:00,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:00,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:00,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:00,954 WARN [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:45:00,954 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a511e45b3024,41027,1732319099972 2024-11-22T23:45:00,956 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T23:45:00,958 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T23:45:00,965 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/hbase.id] with ID: 7c0522d1-e31e-4956-9602-dc143bf990a4 2024-11-22T23:45:00,966 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/.tmp/hbase.id 2024-11-22T23:45:00,972 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/hbase.id] 2024-11-22T23:45:01,014 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:01,019 INFO [master/a511e45b3024:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T23:45:01,039 INFO [master/a511e45b3024:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-22T23:45:01,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,072 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:45:01,075 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T23:45:01,123 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:45:01,130 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:45:01,188 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store 2024-11-22T23:45:01,208 INFO [master/a511e45b3024:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T23:45:01,211 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:01,213 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:45:01,213 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:01,213 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:01,215 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:45:01,215 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:01,215 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:01,216 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732319101212Disabling compacts and flushes for region at 1732319101213 (+1 ms)Disabling writes for close at 1732319101215 (+2 ms)Writing region close event to WAL at 1732319101215Closed at 1732319101215 2024-11-22T23:45:01,216 WARN [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/.initializing 2024-11-22T23:45:01,216 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/WALs/a511e45b3024,41027,1732319099972 2024-11-22T23:45:01,230 INFO [master/a511e45b3024:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:45:01,245 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a511e45b3024%2C41027%2C1732319099972, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/WALs/a511e45b3024,41027,1732319099972, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/oldWALs, maxLogs=10 2024-11-22T23:45:01,286 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/WALs/a511e45b3024,41027,1732319099972/a511e45b3024%2C41027%2C1732319099972.1732319101251 2024-11-22T23:45:01,286 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:45:01,287 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:45:01,288 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:01,291 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,292 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,331 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,361 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T23:45:01,365 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:01,367 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:01,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T23:45:01,373 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:01,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:01,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T23:45:01,379 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:01,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:01,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,386 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T23:45:01,386 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:01,386 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:01,387 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,390 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,391 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,394 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,394 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,397 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:45:01,399 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:01,406 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:45:01,407 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62231009, jitterRate=-0.07268570363521576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:45:01,413 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732319101305Initializing all the Stores at 1732319101306 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319101306Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319101307 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319101307Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319101307Cleaning up temporary data from old regions at 1732319101395 (+88 ms)Region opened successfully at 1732319101413 (+18 ms) 2024-11-22T23:45:01,414 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T23:45:01,447 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3288fea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:45:01,481 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T23:45:01,494 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T23:45:01,494 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T23:45:01,498 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:45:01,498 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:45:01,505 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T23:45:01,505 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T23:45:01,535 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T23:45:01,545 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T23:45:01,548 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T23:45:01,551 INFO [master/a511e45b3024:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T23:45:01,553 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T23:45:01,555 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T23:45:01,557 INFO [master/a511e45b3024:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T23:45:01,560 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T23:45:01,562 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T23:45:01,564 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T23:45:01,565 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T23:45:01,589 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T23:45:01,591 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T23:45:01,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:01,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:01,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:01,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,609 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a511e45b3024,41027,1732319099972, sessionid=0x1014211a5a70000, setting cluster-up flag (Was=false) 2024-11-22T23:45:01,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,632 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T23:45:01,633 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a511e45b3024,41027,1732319099972 2024-11-22T23:45:01,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:01,644 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T23:45:01,645 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a511e45b3024,41027,1732319099972 2024-11-22T23:45:01,651 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T23:45:01,693 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(746): ClusterId : 7c0522d1-e31e-4956-9602-dc143bf990a4 2024-11-22T23:45:01,693 INFO [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(746): ClusterId : 7c0522d1-e31e-4956-9602-dc143bf990a4 2024-11-22T23:45:01,696 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T23:45:01,696 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T23:45:01,702 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T23:45:01,702 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T23:45:01,702 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T23:45:01,702 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T23:45:01,706 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T23:45:01,706 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T23:45:01,706 DEBUG [RS:1;a511e45b3024:34619 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600e9d38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:45:01,707 DEBUG [RS:0;a511e45b3024:33031 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ab92a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:45:01,722 DEBUG [RS:0;a511e45b3024:33031 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a511e45b3024:33031 2024-11-22T23:45:01,724 INFO [RS:0;a511e45b3024:33031 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T23:45:01,725 INFO [RS:0;a511e45b3024:33031 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T23:45:01,725 DEBUG [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T23:45:01,726 DEBUG [RS:1;a511e45b3024:34619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a511e45b3024:34619 2024-11-22T23:45:01,726 INFO [RS:1;a511e45b3024:34619 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T23:45:01,726 INFO [RS:1;a511e45b3024:34619 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T23:45:01,727 DEBUG [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T23:45:01,728 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(2659): reportForDuty to master=a511e45b3024,41027,1732319099972 with port=33031, startcode=1732319100720 2024-11-22T23:45:01,728 INFO [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(2659): reportForDuty to master=a511e45b3024,41027,1732319099972 with port=34619, startcode=1732319100859 2024-11-22T23:45:01,736 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:45:01,742 DEBUG [RS:0;a511e45b3024:33031 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T23:45:01,742 DEBUG [RS:1;a511e45b3024:34619 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T23:45:01,748 INFO [master/a511e45b3024:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T23:45:01,758 INFO [master/a511e45b3024:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T23:45:01,764 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a511e45b3024,41027,1732319099972 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T23:45:01,772 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:45:01,772 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:45:01,773 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:45:01,773 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:45:01,773 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a511e45b3024:0, corePoolSize=10, maxPoolSize=10 2024-11-22T23:45:01,774 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,774 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:45:01,774 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,777 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319131777 2024-11-22T23:45:01,779 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T23:45:01,779 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:45:01,780 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T23:45:01,780 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T23:45:01,784 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T23:45:01,784 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T23:45:01,784 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T23:45:01,784 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T23:45:01,785 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:01,786 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T23:45:01,785 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,797 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T23:45:01,798 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T23:45:01,799 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T23:45:01,803 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T23:45:01,803 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T23:45:01,805 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732319101805,5,FailOnTimeoutGroup] 2024-11-22T23:45:01,809 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732319101806,5,FailOnTimeoutGroup] 2024-11-22T23:45:01,809 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,810 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T23:45:01,811 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,811 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,824 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54929, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T23:45:01,824 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58983, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T23:45:01,827 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T23:45:01,827 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 2024-11-22T23:45:01,831 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41027 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a511e45b3024,33031,1732319100720 2024-11-22T23:45:01,834 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41027 {}] master.ServerManager(517): Registering regionserver=a511e45b3024,33031,1732319100720 2024-11-22T23:45:01,842 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:01,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T23:45:01,847 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41027 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a511e45b3024,34619,1732319100859 2024-11-22T23:45:01,847 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41027 {}] master.ServerManager(517): Registering regionserver=a511e45b3024,34619,1732319100859 2024-11-22T23:45:01,848 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:45:01,848 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:01,849 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:01,849 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T23:45:01,852 DEBUG [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 2024-11-22T23:45:01,852 DEBUG [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 2024-11-22T23:45:01,852 DEBUG [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T23:45:01,852 DEBUG [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T23:45:01,852 DEBUG [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T23:45:01,853 DEBUG [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T23:45:01,855 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:45:01,855 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:01,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:01,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T23:45:01,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:45:01,861 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:45:01,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:01,862 DEBUG [RS:0;a511e45b3024:33031 {}] zookeeper.ZKUtil(111): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a511e45b3024,33031,1732319100720 2024-11-22T23:45:01,862 WARN [RS:0;a511e45b3024:33031 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:45:01,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:01,862 INFO [RS:0;a511e45b3024:33031 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:45:01,862 DEBUG [RS:1;a511e45b3024:34619 {}] zookeeper.ZKUtil(111): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a511e45b3024,34619,1732319100859 2024-11-22T23:45:01,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T23:45:01,862 WARN [RS:1;a511e45b3024:34619 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:45:01,862 DEBUG [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,33031,1732319100720 2024-11-22T23:45:01,863 INFO [RS:1;a511e45b3024:34619 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:45:01,863 DEBUG [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34619,1732319100859 2024-11-22T23:45:01,864 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a511e45b3024,33031,1732319100720] 2024-11-22T23:45:01,864 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a511e45b3024,34619,1732319100859] 2024-11-22T23:45:01,877 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:45:01,877 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:01,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:01,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:45:01,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740 2024-11-22T23:45:01,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740 2024-11-22T23:45:01,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:45:01,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:45:01,880 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:45:01,882 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:45:01,890 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:45:01,890 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68632432, jitterRate=0.022702932357788086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:45:01,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732319101842Initializing all the Stores at 1732319101842Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319101842Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319101842Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319101842Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319101842Cleaning up temporary data from old regions at 1732319101880 (+38 ms)Region opened successfully at 1732319101893 (+13 ms) 2024-11-22T23:45:01,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:45:01,894 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:45:01,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:45:01,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:45:01,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:45:01,895 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:45:01,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732319101894Disabling compacts and flushes for region at 1732319101894Disabling writes for close at 1732319101894Writing region close event to WAL at 1732319101895 (+1 ms)Closed at 1732319101895 2024-11-22T23:45:01,897 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:45:01,897 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T23:45:01,898 INFO [RS:0;a511e45b3024:33031 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T23:45:01,898 INFO [RS:1;a511e45b3024:34619 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T23:45:01,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T23:45:01,912 INFO [RS:0;a511e45b3024:33031 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T23:45:01,912 INFO [RS:1;a511e45b3024:34619 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T23:45:01,913 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:45:01,915 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T23:45:01,918 INFO [RS:0;a511e45b3024:33031 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T23:45:01,918 INFO [RS:1;a511e45b3024:34619 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T23:45:01,918 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,918 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,919 INFO [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T23:45:01,919 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T23:45:01,925 INFO [RS:1;a511e45b3024:34619 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T23:45:01,925 INFO [RS:0;a511e45b3024:33031 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T23:45:01,927 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,927 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,928 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,928 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,928 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,928 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,928 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,928 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,928 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,928 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,929 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,929 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,929 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:45:01,929 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:45:01,929 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,929 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,929 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,929 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,930 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,930 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,930 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,930 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,930 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,930 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,930 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,930 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:01,930 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:45:01,931 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:45:01,931 DEBUG [RS:1;a511e45b3024:34619 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:45:01,931 DEBUG [RS:0;a511e45b3024:33031 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:45:01,933 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,933 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,933 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,933 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,933 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,933 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,933 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,933 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,933 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,933 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,34619,1732319100859-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:45:01,934 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,934 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,33031,1732319100720-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:45:01,960 INFO [RS:1;a511e45b3024:34619 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T23:45:01,963 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,34619,1732319100859-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,963 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,963 INFO [RS:1;a511e45b3024:34619 {}] regionserver.Replication(171): a511e45b3024,34619,1732319100859 started 2024-11-22T23:45:01,969 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T23:45:01,969 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,33031,1732319100720-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,969 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,969 INFO [RS:0;a511e45b3024:33031 {}] regionserver.Replication(171): a511e45b3024,33031,1732319100720 started 2024-11-22T23:45:01,985 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,985 INFO [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(1482): Serving as a511e45b3024,34619,1732319100859, RpcServer on a511e45b3024/172.17.0.2:34619, sessionid=0x1014211a5a70002 2024-11-22T23:45:01,986 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T23:45:01,986 DEBUG [RS:1;a511e45b3024:34619 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a511e45b3024,34619,1732319100859 2024-11-22T23:45:01,986 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,34619,1732319100859' 2024-11-22T23:45:01,986 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T23:45:01,988 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T23:45:01,988 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T23:45:01,988 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T23:45:01,988 DEBUG [RS:1;a511e45b3024:34619 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a511e45b3024,34619,1732319100859 2024-11-22T23:45:01,989 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,34619,1732319100859' 2024-11-22T23:45:01,989 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T23:45:01,989 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T23:45:01,990 DEBUG [RS:1;a511e45b3024:34619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T23:45:01,990 INFO [RS:1;a511e45b3024:34619 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T23:45:01,991 INFO [RS:1;a511e45b3024:34619 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T23:45:01,996 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:01,996 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(1482): Serving as a511e45b3024,33031,1732319100720, RpcServer on a511e45b3024/172.17.0.2:33031, sessionid=0x1014211a5a70001 2024-11-22T23:45:01,996 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T23:45:01,996 DEBUG [RS:0;a511e45b3024:33031 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a511e45b3024,33031,1732319100720 2024-11-22T23:45:01,997 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,33031,1732319100720' 2024-11-22T23:45:01,997 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T23:45:01,997 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T23:45:01,998 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T23:45:01,998 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T23:45:01,998 DEBUG [RS:0;a511e45b3024:33031 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a511e45b3024,33031,1732319100720 2024-11-22T23:45:01,998 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,33031,1732319100720' 2024-11-22T23:45:01,998 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T23:45:01,999 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T23:45:02,000 DEBUG [RS:0;a511e45b3024:33031 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T23:45:02,000 INFO [RS:0;a511e45b3024:33031 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T23:45:02,000 INFO [RS:0;a511e45b3024:33031 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T23:45:02,066 WARN [a511e45b3024:41027 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T23:45:02,096 INFO [RS:1;a511e45b3024:34619 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:45:02,097 INFO [RS:1;a511e45b3024:34619 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a511e45b3024%2C34619%2C1732319100859, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34619,1732319100859, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs, maxLogs=55 2024-11-22T23:45:02,101 INFO [RS:0;a511e45b3024:33031 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:45:02,101 INFO [RS:0;a511e45b3024:33031 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a511e45b3024%2C33031%2C1732319100720, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,33031,1732319100720, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs, maxLogs=55 2024-11-22T23:45:02,119 INFO [RS:1;a511e45b3024:34619 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34619,1732319100859/a511e45b3024%2C34619%2C1732319100859.1732319102100 2024-11-22T23:45:02,120 DEBUG [RS:1;a511e45b3024:34619 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:45:02,122 INFO [RS:0;a511e45b3024:33031 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,33031,1732319100720/a511e45b3024%2C33031%2C1732319100720.1732319102103 2024-11-22T23:45:02,122 DEBUG [RS:0;a511e45b3024:33031 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:45:02,319 DEBUG [a511e45b3024:41027 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-22T23:45:02,331 DEBUG [a511e45b3024:41027 {}] balancer.BalancerClusterState(204): Hosts are {a511e45b3024=0} racks are {/default-rack=0} 2024-11-22T23:45:02,334 DEBUG [a511e45b3024:41027 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T23:45:02,335 DEBUG [a511e45b3024:41027 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T23:45:02,335 DEBUG [a511e45b3024:41027 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T23:45:02,335 DEBUG [a511e45b3024:41027 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T23:45:02,335 INFO [a511e45b3024:41027 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T23:45:02,335 INFO [a511e45b3024:41027 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T23:45:02,335 DEBUG [a511e45b3024:41027 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T23:45:02,342 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a511e45b3024,33031,1732319100720 2024-11-22T23:45:02,348 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,33031,1732319100720, state=OPENING 2024-11-22T23:45:02,360 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T23:45:02,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:02,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:02,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:02,363 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:02,363 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:02,363 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:02,365 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:45:02,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a511e45b3024,33031,1732319100720}] 2024-11-22T23:45:02,540 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T23:45:02,543 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49617, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T23:45:02,551 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T23:45:02,552 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:45:02,552 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T23:45:02,553 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a511e45b3024%2C33031%2C1732319100720.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,33031,1732319100720, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs, maxLogs=55 2024-11-22T23:45:02,576 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,33031,1732319100720/a511e45b3024%2C33031%2C1732319100720.meta.1732319102555.meta 2024-11-22T23:45:02,577 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:45:02,577 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:45:02,579 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T23:45:02,581 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T23:45:02,586 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T23:45:02,590 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T23:45:02,591 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:02,591 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T23:45:02,591 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T23:45:02,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T23:45:02,594 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:45:02,594 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:02,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:02,595 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T23:45:02,595 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:45:02,596 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:02,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:02,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T23:45:02,597 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:45:02,597 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:02,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:02,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T23:45:02,598 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:45:02,598 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:02,599 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:02,599 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:45:02,600 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740 2024-11-22T23:45:02,601 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740 2024-11-22T23:45:02,601 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:45:02,601 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:45:02,601 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:45:02,602 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:45:02,603 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67202188, jitterRate=0.0013906359672546387}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:45:02,603 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T23:45:02,605 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732319102592Writing region info on filesystem at 1732319102592Initializing all the Stores at 1732319102592Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319102592Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319102593 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319102593Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319102593Cleaning up temporary data from old regions at 1732319102601 (+8 ms)Running coprocessor post-open hooks at 1732319102603 (+2 ms)Region opened successfully at 1732319102605 (+2 ms) 2024-11-22T23:45:02,611 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732319102531 2024-11-22T23:45:02,622 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T23:45:02,622 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T23:45:02,623 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,33031,1732319100720 2024-11-22T23:45:02,624 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,33031,1732319100720, state=OPEN 2024-11-22T23:45:02,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:02,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:02,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:02,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:02,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:02,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:02,627 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a511e45b3024,33031,1732319100720 2024-11-22T23:45:02,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T23:45:02,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a511e45b3024,33031,1732319100720 in 260 msec 2024-11-22T23:45:02,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T23:45:02,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 728 msec 2024-11-22T23:45:02,636 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:45:02,636 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T23:45:02,657 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:45:02,658 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,33031,1732319100720, seqNum=-1] 2024-11-22T23:45:02,679 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:45:02,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59651, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:45:02,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0170 sec 2024-11-22T23:45:02,697 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732319102697, completionTime=-1 2024-11-22T23:45:02,700 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-22T23:45:02,700 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:45:02,726 INFO [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T23:45:02,726 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319162726 2024-11-22T23:45:02,726 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319222726 2024-11-22T23:45:02,726 INFO [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-22T23:45:02,729 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-22T23:45:02,734 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41027,1732319099972-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:02,735 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41027,1732319099972-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:02,735 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41027,1732319099972-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:02,736 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a511e45b3024:41027, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:02,737 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:02,737 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:02,743 DEBUG [master/a511e45b3024:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T23:45:02,764 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.810sec 2024-11-22T23:45:02,766 INFO [master/a511e45b3024:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T23:45:02,767 INFO [master/a511e45b3024:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T23:45:02,768 INFO [master/a511e45b3024:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T23:45:02,768 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T23:45:02,768 INFO [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T23:45:02,769 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41027,1732319099972-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:45:02,770 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41027,1732319099972-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T23:45:02,771 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T23:45:02,772 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T23:45:02,772 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41027,1732319099972-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:02,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8eba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:45:02,822 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T23:45:02,823 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T23:45:02,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,41027,-1 for getting cluster id 2024-11-22T23:45:02,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:45:02,837 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7c0522d1-e31e-4956-9602-dc143bf990a4' 2024-11-22T23:45:02,840 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:45:02,840 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7c0522d1-e31e-4956-9602-dc143bf990a4" 2024-11-22T23:45:02,840 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a1f572, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:45:02,840 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,41027,-1] 2024-11-22T23:45:02,843 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:45:02,845 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:02,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:45:02,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:45:02,850 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:45:02,857 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,33031,1732319100720, seqNum=-1] 2024-11-22T23:45:02,857 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:45:02,860 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:45:02,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a511e45b3024,41027,1732319099972 2024-11-22T23:45:02,883 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:45:02,887 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a511e45b3024,41027,1732319099972 2024-11-22T23:45:02,889 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a2d937d 2024-11-22T23:45:02,890 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:45:02,892 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:45:02,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732319102879', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:45:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732319102879 2024-11-22T23:45:02,906 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732319102879 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:45:02,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732319102879" procId is: 4 2024-11-22T23:45:02,909 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:02,910 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732319102879 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:45:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:45:02,924 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732319102879-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7aa33722577c859ad5925cbaed669e0b, NAME => 'testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732319102879', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 2024-11-22T23:45:02,930 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732319102879-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:02,930 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732319102879-pool-0 {}] regionserver.HRegion(1722): Closing 7aa33722577c859ad5925cbaed669e0b, disabling compactions & flushes 2024-11-22T23:45:02,930 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732319102879-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:02,931 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732319102879-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:02,931 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732319102879-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. after waiting 0 ms 2024-11-22T23:45:02,931 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732319102879-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:02,931 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732319102879-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:02,931 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732319102879-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7aa33722577c859ad5925cbaed669e0b: Waiting for close lock at 1732319102930Disabling compacts and flushes for region at 1732319102930Disabling writes for close at 1732319102931 (+1 ms)Writing region close event to WAL at 1732319102931Closed at 1732319102931 2024-11-22T23:45:02,932 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732319102879 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:45:02,937 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732319102932"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732319102932"}]},"ts":"1732319102932"} 2024-11-22T23:45:02,942 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:45:02,943 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732319102879 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:45:02,947 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732319102879","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319102944"}]},"ts":"1732319102944"} 2024-11-22T23:45:02,951 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732319102879, state=ENABLING in hbase:meta 2024-11-22T23:45:02,952 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a511e45b3024=0} racks are {/default-rack=0} 2024-11-22T23:45:02,952 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T23:45:02,952 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T23:45:02,952 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T23:45:02,952 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T23:45:02,952 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T23:45:02,952 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T23:45:02,952 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T23:45:02,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, ASSIGN}] 2024-11-22T23:45:02,956 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, ASSIGN 2024-11-22T23:45:02,957 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, ASSIGN; state=OFFLINE, location=a511e45b3024,34619,1732319100859; forceNewPlan=false, retain=false 2024-11-22T23:45:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:45:03,109 INFO [a511e45b3024:41027 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T23:45:03,110 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7aa33722577c859ad5925cbaed669e0b, regionState=OPENING, regionLocation=a511e45b3024,34619,1732319100859 2024-11-22T23:45:03,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, ASSIGN because future has completed 2024-11-22T23:45:03,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa33722577c859ad5925cbaed669e0b, server=a511e45b3024,34619,1732319100859}] 2024-11-22T23:45:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:45:03,268 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T23:45:03,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38765, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T23:45:03,272 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:03,273 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7aa33722577c859ad5925cbaed669e0b, NAME => 'testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:45:03,273 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732319102879 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,273 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:03,273 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,273 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,274 INFO [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,279 INFO [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa33722577c859ad5925cbaed669e0b columnFamilyName family 2024-11-22T23:45:03,279 DEBUG [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:03,279 INFO [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] regionserver.HStore(327): Store=7aa33722577c859ad5925cbaed669e0b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:03,279 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,280 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,280 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,280 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,281 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,281 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,288 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:45:03,288 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7aa33722577c859ad5925cbaed669e0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64160403, jitterRate=-0.043935492634773254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T23:45:03,288 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:03,289 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7aa33722577c859ad5925cbaed669e0b: Running coprocessor pre-open hook at 1732319103274Writing region info on filesystem at 1732319103274Initializing all the Stores at 1732319103274Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319103274Cleaning up temporary data from old regions at 1732319103281 (+7 ms)Running coprocessor post-open hooks at 1732319103288 (+7 ms)Region opened successfully at 1732319103289 (+1 ms) 2024-11-22T23:45:03,290 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b., pid=6, masterSystemTime=1732319103267 2024-11-22T23:45:03,293 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:03,293 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:03,294 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7aa33722577c859ad5925cbaed669e0b, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,34619,1732319100859 2024-11-22T23:45:03,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa33722577c859ad5925cbaed669e0b, server=a511e45b3024,34619,1732319100859 because future has completed 2024-11-22T23:45:03,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T23:45:03,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7aa33722577c859ad5925cbaed669e0b, server=a511e45b3024,34619,1732319100859 in 184 msec 2024-11-22T23:45:03,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T23:45:03,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, ASSIGN in 346 msec 2024-11-22T23:45:03,302 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732319102879 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:45:03,303 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732319102879","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319103303"}]},"ts":"1732319103303"} 2024-11-22T23:45:03,305 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732319102879, state=ENABLED in hbase:meta 2024-11-22T23:45:03,306 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732319102879 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:45:03,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732319102879 in 406 msec 2024-11-22T23:45:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:45:03,544 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732319102879 get assigned. Timeout = 60000ms 2024-11-22T23:45:03,544 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732319102879 completed 2024-11-22T23:45:03,547 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T23:45:03,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732319102879 assigned to meta. Checking AM states. 2024-11-22T23:45:03,553 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T23:45:03,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732319102879 assigned. 2024-11-22T23:45:03,562 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732319102879', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b., hostname=a511e45b3024,34619,1732319100859, seqNum=2] 2024-11-22T23:45:03,564 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:45:03,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:45:03,576 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7aa33722577c859ad5925cbaed669e0b 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T23:45:03,601 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fc2db46b8063446ab246e333b7869930 2024-11-22T23:45:03,638 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/.tmp/family/fc2db46b8063446ab246e333b7869930 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T23:45:03,643 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/.tmp/family/fc2db46b8063446ab246e333b7869930 2024-11-22T23:45:03,679 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/.tmp/family/fc2db46b8063446ab246e333b7869930 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930 2024-11-22T23:45:03,681 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930, delay=912 ms 2024-11-22T23:45:03,683 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T23:45:03,688 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 7aa33722577c859ad5925cbaed669e0b in 112ms, sequenceid=6, compaction requested=false 2024-11-22T23:45:03,690 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732319102879' 2024-11-22T23:45:03,691 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7aa33722577c859ad5925cbaed669e0b: 2024-11-22T23:45:04,595 DEBUG [hfile-prefetch-1732319103681 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fc2db46b8063446ab246e333b7869930_0. 2024-11-22T23:45:04,599 DEBUG [hfile-prefetch-1732319103681 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fc2db46b8063446ab246e333b7869930_117. 2024-11-22T23:45:04,600 DEBUG [hfile-prefetch-1732319103681 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930, with total blocks 2, and data blocks 1 2024-11-22T23:45:04,600 DEBUG [hfile-prefetch-1732319103681 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T23:45:04,601 DEBUG [hfile-prefetch-1732319103681 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T23:45:04,601 DEBUG [hfile-prefetch-1732319103681 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930. Will sleep 100ms and try the verification again. 2024-11-22T23:45:04,610 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T23:45:04,610 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T23:45:04,611 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T23:45:04,611 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T23:45:04,626 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T23:45:04,626 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732319104610 2024-11-22T23:45:04,626 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T23:45:04,701 DEBUG [hfile-prefetch-1732319103681 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930, with total blocks 2, and data blocks 1 2024-11-22T23:45:04,701 DEBUG [hfile-prefetch-1732319103681 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-22T23:45:04,702 DEBUG [hfile-prefetch-1732319103681 {}] bucket.BucketCache(2293): found block for file fc2db46b8063446ab246e333b7869930 in the backing map. Acquiring read lock for offset 0 2024-11-22T23:45:04,702 DEBUG [hfile-prefetch-1732319103681 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930 has now been fully cached. 2024-11-22T23:45:04,702 DEBUG [hfile-prefetch-1732319103681 {}] hfile.PrefetchExecutor(121): Prefetch completed for fc2db46b8063446ab246e333b7869930 2024-11-22T23:45:04,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=7aa33722577c859ad5925cbaed669e0b, source=a511e45b3024,34619,1732319100859, destination=a511e45b3024,33031,1732319100720, warming up region on a511e45b3024,33031,1732319100720 2024-11-22T23:45:04,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=7aa33722577c859ad5925cbaed669e0b, source=a511e45b3024,34619,1732319100859, destination=a511e45b3024,33031,1732319100720, running balancer 2024-11-22T23:45:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, REOPEN/MOVE 2024-11-22T23:45:04,706 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, REOPEN/MOVE 2024-11-22T23:45:04,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33031 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33031 {}] regionserver.HRegion(7855): Warmup {ENCODED => 7aa33722577c859ad5925cbaed669e0b, NAME => 'testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:45:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33031 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:04,707 INFO [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:04,707 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7aa33722577c859ad5925cbaed669e0b, regionState=CLOSING, regionLocation=a511e45b3024,34619,1732319100859 2024-11-22T23:45:04,708 INFO [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa33722577c859ad5925cbaed669e0b columnFamilyName family 2024-11-22T23:45:04,708 DEBUG [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:04,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, REOPEN/MOVE because future has completed 2024-11-22T23:45:04,711 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-22T23:45:04,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7aa33722577c859ad5925cbaed669e0b, server=a511e45b3024,34619,1732319100859}] 2024-11-22T23:45:04,711 DEBUG [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930 2024-11-22T23:45:04,711 INFO [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] regionserver.HStore(327): Store=7aa33722577c859ad5925cbaed669e0b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33031 {}] regionserver.HRegion(1722): Closing 7aa33722577c859ad5925cbaed669e0b, disabling compactions & flushes 2024-11-22T23:45:04,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33031 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33031 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33031 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. after waiting 0 ms 2024-11-22T23:45:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33031 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:04,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33031 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33031 {}] regionserver.HRegion(1676): Region close journal for 7aa33722577c859ad5925cbaed669e0b: Waiting for close lock at 1732319104712Disabling compacts and flushes for region at 1732319104712Disabling writes for close at 1732319104712Writing region close event to WAL at 1732319104713 (+1 ms)Closed at 1732319104713 2024-11-22T23:45:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41027 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-22T23:45:04,869 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:04,869 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-22T23:45:04,870 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 7aa33722577c859ad5925cbaed669e0b, disabling compactions & flushes 2024-11-22T23:45:04,870 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:04,870 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:04,870 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. after waiting 0 ms 2024-11-22T23:45:04,870 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:04,872 DEBUG [StoreFileCloser-7aa33722577c859ad5925cbaed669e0b-family-1 {}] bucket.BucketCache(768): Removing file fc2db46b8063446ab246e333b7869930 for region 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:04,872 DEBUG [StoreFileCloser-7aa33722577c859ad5925cbaed669e0b-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file fc2db46b8063446ab246e333b7869930, starting offset: 0, end offset: 9223372036854775807 2024-11-22T23:45:04,873 DEBUG [StoreFileCloser-7aa33722577c859ad5925cbaed669e0b-family-1 {}] bucket.BucketCache(856): removed key fc2db46b8063446ab246e333b7869930_0 from back map with offset lock 2094080 in the evict process 2024-11-22T23:45:04,878 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T23:45:04,878 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T23:45:04,878 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T23:45:04,878 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T23:45:04,879 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T23:45:04,879 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:04,879 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 7aa33722577c859ad5925cbaed669e0b: Waiting for close lock at 1732319104870Running coprocessor pre-close hooks at 1732319104870Disabling compacts and flushes for region at 1732319104870Disabling writes for close at 1732319104870Writing region close event to WAL at 1732319104874 (+4 ms)Running coprocessor post-close hooks at 1732319104879 (+5 ms)Closed at 1732319104879 2024-11-22T23:45:04,880 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 7aa33722577c859ad5925cbaed669e0b move to a511e45b3024,33031,1732319100720 record at close sequenceid=6 2024-11-22T23:45:04,881 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T23:45:04,882 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732319104878 2024-11-22T23:45:04,882 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T23:45:04,883 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:04,884 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7aa33722577c859ad5925cbaed669e0b, regionState=CLOSED 2024-11-22T23:45:04,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7aa33722577c859ad5925cbaed669e0b, server=a511e45b3024,34619,1732319100859 because future has completed 2024-11-22T23:45:04,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T23:45:04,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 7aa33722577c859ad5925cbaed669e0b, server=a511e45b3024,34619,1732319100859 in 176 msec 2024-11-22T23:45:04,890 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, REOPEN/MOVE; state=CLOSED, location=a511e45b3024,33031,1732319100720; forceNewPlan=false, retain=false 2024-11-22T23:45:05,040 INFO [a511e45b3024:41027 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T23:45:05,041 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7aa33722577c859ad5925cbaed669e0b, regionState=OPENING, regionLocation=a511e45b3024,33031,1732319100720 2024-11-22T23:45:05,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, REOPEN/MOVE because future has completed 2024-11-22T23:45:05,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa33722577c859ad5925cbaed669e0b, server=a511e45b3024,33031,1732319100720}] 2024-11-22T23:45:05,200 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:05,201 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 7aa33722577c859ad5925cbaed669e0b, NAME => 'testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:45:05,201 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732319102879 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,201 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:05,202 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,202 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,203 INFO [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,204 INFO [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa33722577c859ad5925cbaed669e0b columnFamilyName family 2024-11-22T23:45:05,204 DEBUG [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:05,206 DEBUG [StoreFileOpener-7aa33722577c859ad5925cbaed669e0b-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930, delay=976 ms 2024-11-22T23:45:05,207 DEBUG [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930 2024-11-22T23:45:05,207 INFO [StoreOpener-7aa33722577c859ad5925cbaed669e0b-1 {}] regionserver.HStore(327): Store=7aa33722577c859ad5925cbaed669e0b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:05,208 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,209 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,210 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,210 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,210 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,216 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,217 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 7aa33722577c859ad5925cbaed669e0b; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75287967, jitterRate=0.12187813222408295}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T23:45:05,217 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,219 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 7aa33722577c859ad5925cbaed669e0b: Running coprocessor pre-open hook at 1732319105202Writing region info on filesystem at 1732319105202Initializing all the Stores at 1732319105202Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319105202Cleaning up temporary data from old regions at 1732319105210 (+8 ms)Running coprocessor post-open hooks at 1732319105217 (+7 ms)Region opened successfully at 1732319105219 (+2 ms) 2024-11-22T23:45:05,220 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b., pid=9, masterSystemTime=1732319105197 2024-11-22T23:45:05,223 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:05,223 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:05,223 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7aa33722577c859ad5925cbaed669e0b, regionState=OPEN, openSeqNum=10, regionLocation=a511e45b3024,33031,1732319100720 2024-11-22T23:45:05,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa33722577c859ad5925cbaed669e0b, server=a511e45b3024,33031,1732319100720 because future has completed 2024-11-22T23:45:05,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-22T23:45:05,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7aa33722577c859ad5925cbaed669e0b, server=a511e45b3024,33031,1732319100720 in 182 msec 2024-11-22T23:45:05,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732319102879, region=7aa33722577c859ad5925cbaed669e0b, REOPEN/MOVE in 524 msec 2024-11-22T23:45:05,258 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T23:45:05,259 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:45:05,259 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:45:05,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:05,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:05,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:45:05,265 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T23:45:05,265 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-22T23:45:05,265 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a511e45b3024,41027,1732319099972 2024-11-22T23:45:05,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:05,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:05,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:05,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:05,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:05,268 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:45:05,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:05,269 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:45:05,269 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:45:05,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:05,269 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a511e45b3024,33031,1732319100720' ***** 2024-11-22T23:45:05,269 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T23:45:05,270 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a511e45b3024,34619,1732319100859' ***** 2024-11-22T23:45:05,270 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T23:45:05,270 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:05,270 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:05,270 INFO [RS:0;a511e45b3024:33031 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T23:45:05,270 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:05,270 INFO [RS:1;a511e45b3024:34619 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T23:45:05,270 INFO [RS:1;a511e45b3024:34619 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/bucket.persistence 2024-11-22T23:45:05,270 INFO [RS:0;a511e45b3024:33031 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/bucket.persistence 2024-11-22T23:45:05,271 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T23:45:05,271 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T23:45:05,271 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T23:45:05,271 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T23:45:05,271 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T23:45:05,271 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T23:45:05,271 INFO [RS:1;a511e45b3024:34619 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T23:45:05,271 INFO [RS:0;a511e45b3024:33031 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T23:45:05,271 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T23:45:05,271 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T23:45:05,281 DEBUG [RS:1;a511e45b3024:34619 {}] bucket.BucketCache(1487): Thread RS:1;a511e45b3024:34619 started persisting bucket cache to file 2024-11-22T23:45:05,282 DEBUG [RS:0;a511e45b3024:33031 {}] bucket.BucketCache(1487): Thread RS:0;a511e45b3024:33031 started persisting bucket cache to file 2024-11-22T23:45:05,282 DEBUG [RS:1;a511e45b3024:34619 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T23:45:05,282 DEBUG [RS:1;a511e45b3024:34619 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T23:45:05,282 DEBUG [RS:0;a511e45b3024:33031 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T23:45:05,282 DEBUG [RS:0;a511e45b3024:33031 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T23:45:05,285 DEBUG [RS:0;a511e45b3024:33031 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T23:45:05,285 DEBUG [RS:0;a511e45b3024:33031 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732319105282 2024-11-22T23:45:05,285 DEBUG [RS:0;a511e45b3024:33031 {}] bucket.BucketCache(1509): Thread RS:0;a511e45b3024:33031 finished persisting bucket cache to file, renaming 2024-11-22T23:45:05,285 DEBUG [RS:1;a511e45b3024:34619 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T23:45:05,286 DEBUG [RS:1;a511e45b3024:34619 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732319105281 2024-11-22T23:45:05,286 DEBUG [RS:1;a511e45b3024:34619 {}] bucket.BucketCache(1509): Thread RS:1;a511e45b3024:34619 finished persisting bucket cache to file, renaming 2024-11-22T23:45:05,286 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T23:45:05,286 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T23:45:05,286 INFO [RS:0;a511e45b3024:33031 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T23:45:05,286 INFO [RS:0;a511e45b3024:33031 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T23:45:05,286 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(3091): Received CLOSE for 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,286 INFO [RS:1;a511e45b3024:34619 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T23:45:05,286 INFO [RS:1;a511e45b3024:34619 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T23:45:05,286 INFO [RS:1;a511e45b3024:34619 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T23:45:05,286 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T23:45:05,287 INFO [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(959): stopping server a511e45b3024,34619,1732319100859 2024-11-22T23:45:05,287 INFO [RS:1;a511e45b3024:34619 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:45:05,287 INFO [RS:1;a511e45b3024:34619 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a511e45b3024:34619. 2024-11-22T23:45:05,287 DEBUG [RS:1;a511e45b3024:34619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:45:05,287 DEBUG [RS:1;a511e45b3024:34619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:05,287 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(959): stopping server a511e45b3024,33031,1732319100720 2024-11-22T23:45:05,287 INFO [RS:0;a511e45b3024:33031 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:45:05,287 INFO [RS:0;a511e45b3024:33031 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a511e45b3024:33031. 2024-11-22T23:45:05,287 INFO [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(976): stopping server a511e45b3024,34619,1732319100859; all regions closed. 2024-11-22T23:45:05,287 DEBUG [RS:0;a511e45b3024:33031 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:45:05,287 DEBUG [RS:0;a511e45b3024:33031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:05,287 INFO [RS:0;a511e45b3024:33031 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T23:45:05,287 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T23:45:05,288 INFO [RS:0;a511e45b3024:33031 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T23:45:05,288 INFO [RS:0;a511e45b3024:33031 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T23:45:05,288 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T23:45:05,288 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7aa33722577c859ad5925cbaed669e0b, disabling compactions & flushes 2024-11-22T23:45:05,288 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:05,288 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:05,288 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. after waiting 0 ms 2024-11-22T23:45:05,288 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-22T23:45:05,288 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:05,288 DEBUG [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(1325): Online Regions={7aa33722577c859ad5925cbaed669e0b=testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b., 1588230740=hbase:meta,,1.1588230740} 2024-11-22T23:45:05,288 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T23:45:05,289 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:45:05,289 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:45:05,289 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:45:05,289 DEBUG [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7aa33722577c859ad5925cbaed669e0b 2024-11-22T23:45:05,289 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:45:05,289 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:45:05,289 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-22T23:45:05,289 DEBUG [StoreFileCloser-7aa33722577c859ad5925cbaed669e0b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/family/fc2db46b8063446ab246e333b7869930 2024-11-22T23:45:05,293 DEBUG [RS:1;a511e45b3024:34619 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs 2024-11-22T23:45:05,293 INFO [RS:1;a511e45b3024:34619 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C34619%2C1732319100859:(num 1732319102100) 2024-11-22T23:45:05,293 DEBUG [RS:1;a511e45b3024:34619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:05,293 INFO [RS:1;a511e45b3024:34619 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:45:05,293 INFO [RS:1;a511e45b3024:34619 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:45:05,293 INFO [RS:1;a511e45b3024:34619 {}] hbase.ChoreService(370): Chore service for: regionserver/a511e45b3024:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T23:45:05,294 INFO [RS:1;a511e45b3024:34619 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T23:45:05,294 INFO [RS:1;a511e45b3024:34619 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T23:45:05,294 INFO [RS:1;a511e45b3024:34619 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T23:45:05,294 INFO [regionserver/a511e45b3024:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:45:05,294 INFO [RS:1;a511e45b3024:34619 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:45:05,294 INFO [RS:1;a511e45b3024:34619 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34619 2024-11-22T23:45:05,298 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnRegionMove1732319102879/7aa33722577c859ad5925cbaed669e0b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-22T23:45:05,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a511e45b3024,34619,1732319100859 2024-11-22T23:45:05,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:45:05,299 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:05,299 INFO [RS:1;a511e45b3024:34619 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:45:05,299 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7aa33722577c859ad5925cbaed669e0b: Waiting for close lock at 1732319105287Running coprocessor pre-close hooks at 1732319105288 (+1 ms)Disabling compacts and flushes for region at 1732319105288Disabling writes for close at 1732319105288Writing region close event to WAL at 1732319105290 (+2 ms)Running coprocessor post-close hooks at 1732319105298 (+8 ms)Closed at 1732319105298 2024-11-22T23:45:05,299 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b. 2024-11-22T23:45:05,302 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 404aeeb87f85482d952303cbbedb42d9 2024-11-22T23:45:05,302 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a511e45b3024,34619,1732319100859] 2024-11-22T23:45:05,304 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a511e45b3024,34619,1732319100859 already deleted, retry=false 2024-11-22T23:45:05,304 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a511e45b3024,34619,1732319100859 expired; onlineServers=1 2024-11-22T23:45:05,320 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/info/404aeeb87f85482d952303cbbedb42d9 is 205, key is testBlockEvictionOnRegionMove1732319102879,,1732319102894.7aa33722577c859ad5925cbaed669e0b./info:regioninfo/1732319105223/Put/seqid=0 2024-11-22T23:45:05,321 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/info/404aeeb87f85482d952303cbbedb42d9 2024-11-22T23:45:05,330 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7a69fbb7698c42adb349843b1a05b4d2 2024-11-22T23:45:05,339 INFO [regionserver/a511e45b3024:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:45:05,345 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/ns/7a69fbb7698c42adb349843b1a05b4d2 is 43, key is default/ns:d/1732319102685/Put/seqid=0 2024-11-22T23:45:05,345 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/ns/7a69fbb7698c42adb349843b1a05b4d2 2024-11-22T23:45:05,346 INFO [regionserver/a511e45b3024:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:45:05,356 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 49603e57a5794182bd04cf35d5b51d8a 2024-11-22T23:45:05,371 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/table/49603e57a5794182bd04cf35d5b51d8a is 78, key is testBlockEvictionOnRegionMove1732319102879/table:state/1732319103303/Put/seqid=0 2024-11-22T23:45:05,371 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/table/49603e57a5794182bd04cf35d5b51d8a 2024-11-22T23:45:05,374 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/info/404aeeb87f85482d952303cbbedb42d9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/404aeeb87f85482d952303cbbedb42d9 2024-11-22T23:45:05,375 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/404aeeb87f85482d952303cbbedb42d9, delay=916 ms 2024-11-22T23:45:05,377 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/404aeeb87f85482d952303cbbedb42d9, entries=15, sequenceid=15, filesize=7.6 K 2024-11-22T23:45:05,377 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/ns/7a69fbb7698c42adb349843b1a05b4d2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/7a69fbb7698c42adb349843b1a05b4d2 2024-11-22T23:45:05,378 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/7a69fbb7698c42adb349843b1a05b4d2, delay=969 ms 2024-11-22T23:45:05,380 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/7a69fbb7698c42adb349843b1a05b4d2, entries=2, sequenceid=15, filesize=5.0 K 2024-11-22T23:45:05,380 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/table/49603e57a5794182bd04cf35d5b51d8a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/49603e57a5794182bd04cf35d5b51d8a 2024-11-22T23:45:05,382 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/49603e57a5794182bd04cf35d5b51d8a, delay=933 ms 2024-11-22T23:45:05,383 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/49603e57a5794182bd04cf35d5b51d8a, entries=2, sequenceid=15, filesize=5.3 K 2024-11-22T23:45:05,384 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=15, compaction requested=false 2024-11-22T23:45:05,384 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T23:45:05,386 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/404aeeb87f85482d952303cbbedb42d9 2024-11-22T23:45:05,386 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/7a69fbb7698c42adb349843b1a05b4d2 2024-11-22T23:45:05,387 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/49603e57a5794182bd04cf35d5b51d8a 2024-11-22T23:45:05,392 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-22T23:45:05,392 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T23:45:05,393 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:45:05,393 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732319105288Running coprocessor pre-close hooks at 1732319105288Disabling compacts and flushes for region at 1732319105288Disabling writes for close at 1732319105289 (+1 ms)Obtaining lock to block concurrent updates at 1732319105289Preparing flush snapshotting stores in 1588230740 at 1732319105289Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732319105290 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732319105290Flushing 1588230740/info: creating writer at 1732319105291 (+1 ms)Flushing 1588230740/info: appending metadata at 1732319105319 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732319105319Flushing 1588230740/ns: creating writer at 1732319105323 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732319105344 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732319105344Flushing 1588230740/table: creating writer at 1732319105349 (+5 ms)Flushing 1588230740/table: appending metadata at 1732319105369 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732319105369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72300ed4: reopening flushed file at 1732319105374 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fa5d72b: reopening flushed file at 1732319105377 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76a70c65: reopening flushed file at 1732319105380 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=15, compaction requested=false at 1732319105384 (+4 ms)Writing region close event to WAL at 1732319105387 (+3 ms)Running coprocessor post-close hooks at 1732319105392 (+5 ms)Closed at 1732319105393 (+1 ms) 2024-11-22T23:45:05,393 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T23:45:05,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:05,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34619-0x1014211a5a70002, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:05,403 INFO [RS:1;a511e45b3024:34619 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:45:05,403 INFO [RS:1;a511e45b3024:34619 {}] regionserver.HRegionServer(1031): Exiting; stopping=a511e45b3024,34619,1732319100859; zookeeper connection closed. 2024-11-22T23:45:05,403 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11197734 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11197734 2024-11-22T23:45:05,489 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(976): stopping server a511e45b3024,33031,1732319100720; all regions closed. 2024-11-22T23:45:05,491 DEBUG [RS:0;a511e45b3024:33031 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs 2024-11-22T23:45:05,491 INFO [RS:0;a511e45b3024:33031 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C33031%2C1732319100720.meta:.meta(num 1732319102555) 2024-11-22T23:45:05,493 DEBUG [RS:0;a511e45b3024:33031 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs 2024-11-22T23:45:05,493 INFO [RS:0;a511e45b3024:33031 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C33031%2C1732319100720:(num 1732319102103) 2024-11-22T23:45:05,493 DEBUG [RS:0;a511e45b3024:33031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:05,493 INFO [RS:0;a511e45b3024:33031 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:45:05,494 INFO [RS:0;a511e45b3024:33031 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:45:05,494 INFO [RS:0;a511e45b3024:33031 {}] hbase.ChoreService(370): Chore service for: regionserver/a511e45b3024:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T23:45:05,494 INFO [RS:0;a511e45b3024:33031 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:45:05,494 INFO [regionserver/a511e45b3024:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:45:05,494 INFO [RS:0;a511e45b3024:33031 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33031 2024-11-22T23:45:05,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:45:05,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a511e45b3024,33031,1732319100720 2024-11-22T23:45:05,497 INFO [RS:0;a511e45b3024:33031 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:45:05,498 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a511e45b3024,33031,1732319100720] 2024-11-22T23:45:05,501 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a511e45b3024,33031,1732319100720 already deleted, retry=false 2024-11-22T23:45:05,501 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a511e45b3024,33031,1732319100720 expired; onlineServers=0 2024-11-22T23:45:05,501 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a511e45b3024,41027,1732319099972' ***** 2024-11-22T23:45:05,501 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T23:45:05,501 INFO [M:0;a511e45b3024:41027 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:45:05,501 INFO [M:0;a511e45b3024:41027 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:45:05,502 DEBUG [M:0;a511e45b3024:41027 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T23:45:05,502 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T23:45:05,502 DEBUG [M:0;a511e45b3024:41027 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T23:45:05,502 DEBUG [master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732319101805 {}] cleaner.HFileCleaner(306): Exit Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732319101805,5,FailOnTimeoutGroup] 2024-11-22T23:45:05,502 DEBUG [master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732319101806 {}] cleaner.HFileCleaner(306): Exit Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732319101806,5,FailOnTimeoutGroup] 2024-11-22T23:45:05,502 INFO [M:0;a511e45b3024:41027 {}] hbase.ChoreService(370): Chore service for: master/a511e45b3024:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T23:45:05,502 INFO [M:0;a511e45b3024:41027 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:45:05,502 DEBUG [M:0;a511e45b3024:41027 {}] master.HMaster(1795): Stopping service threads 2024-11-22T23:45:05,502 INFO [M:0;a511e45b3024:41027 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T23:45:05,503 INFO [M:0;a511e45b3024:41027 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:45:05,503 INFO [M:0;a511e45b3024:41027 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:45:05,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T23:45:05,504 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T23:45:05,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:05,504 DEBUG [M:0;a511e45b3024:41027 {}] zookeeper.ZKUtil(347): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T23:45:05,504 WARN [M:0;a511e45b3024:41027 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T23:45:05,505 INFO [M:0;a511e45b3024:41027 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/.lastflushedseqids 2024-11-22T23:45:05,521 INFO [M:0;a511e45b3024:41027 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:45:05,521 INFO [M:0;a511e45b3024:41027 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T23:45:05,522 DEBUG [M:0;a511e45b3024:41027 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:45:05,522 INFO [M:0;a511e45b3024:41027 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:05,522 DEBUG [M:0;a511e45b3024:41027 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:05,522 DEBUG [M:0;a511e45b3024:41027 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:45:05,522 DEBUG [M:0;a511e45b3024:41027 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:05,523 INFO [M:0;a511e45b3024:41027 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-22T23:45:05,534 DEBUG [M:0;a511e45b3024:41027 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c25f6afea9cc4722bf3d6fd1e7c18437 2024-11-22T23:45:05,554 DEBUG [M:0;a511e45b3024:41027 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c25f6afea9cc4722bf3d6fd1e7c18437 is 82, key is hbase:meta,,1/info:regioninfo/1732319102622/Put/seqid=0 2024-11-22T23:45:05,555 INFO [M:0;a511e45b3024:41027 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c25f6afea9cc4722bf3d6fd1e7c18437 2024-11-22T23:45:05,565 DEBUG [M:0;a511e45b3024:41027 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 41bd98970bab4e7e85548ab7d71220ba 2024-11-22T23:45:05,585 DEBUG [M:0;a511e45b3024:41027 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41bd98970bab4e7e85548ab7d71220ba is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732319103307/Put/seqid=0 2024-11-22T23:45:05,586 INFO [M:0;a511e45b3024:41027 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41bd98970bab4e7e85548ab7d71220ba 2024-11-22T23:45:05,595 DEBUG [M:0;a511e45b3024:41027 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c156660072ab4197b0cb53871bbbaad9 2024-11-22T23:45:05,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:05,599 INFO [RS:0;a511e45b3024:33031 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:45:05,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33031-0x1014211a5a70001, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:05,599 INFO [RS:0;a511e45b3024:33031 {}] regionserver.HRegionServer(1031): Exiting; stopping=a511e45b3024,33031,1732319100720; zookeeper connection closed. 2024-11-22T23:45:05,599 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5e2d39f8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5e2d39f8 2024-11-22T23:45:05,600 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-22T23:45:05,608 DEBUG [M:0;a511e45b3024:41027 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c156660072ab4197b0cb53871bbbaad9 is 69, key is a511e45b3024,33031,1732319100720/rs:state/1732319101837/Put/seqid=0 2024-11-22T23:45:05,608 INFO [M:0;a511e45b3024:41027 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c156660072ab4197b0cb53871bbbaad9 2024-11-22T23:45:05,610 DEBUG [M:0;a511e45b3024:41027 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c25f6afea9cc4722bf3d6fd1e7c18437 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c25f6afea9cc4722bf3d6fd1e7c18437 2024-11-22T23:45:05,613 INFO [M:0;a511e45b3024:41027 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c25f6afea9cc4722bf3d6fd1e7c18437, entries=8, sequenceid=84, filesize=5.5 K 2024-11-22T23:45:05,613 DEBUG [M:0;a511e45b3024:41027 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41bd98970bab4e7e85548ab7d71220ba as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41bd98970bab4e7e85548ab7d71220ba 2024-11-22T23:45:05,616 INFO [M:0;a511e45b3024:41027 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41bd98970bab4e7e85548ab7d71220ba, entries=9, sequenceid=84, filesize=6.5 K 2024-11-22T23:45:05,616 DEBUG [M:0;a511e45b3024:41027 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c156660072ab4197b0cb53871bbbaad9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c156660072ab4197b0cb53871bbbaad9 2024-11-22T23:45:05,618 INFO [M:0;a511e45b3024:41027 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c156660072ab4197b0cb53871bbbaad9, entries=2, sequenceid=84, filesize=5.1 K 2024-11-22T23:45:05,619 INFO [M:0;a511e45b3024:41027 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 96ms, sequenceid=84, compaction requested=false 2024-11-22T23:45:05,621 INFO [M:0;a511e45b3024:41027 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:05,621 DEBUG [M:0;a511e45b3024:41027 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732319105522Disabling compacts and flushes for region at 1732319105522Disabling writes for close at 1732319105522Obtaining lock to block concurrent updates at 1732319105523 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732319105523Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1732319105523Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732319105524 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732319105524Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732319105554 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732319105554Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732319105558 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732319105584 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732319105585 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732319105588 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732319105607 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732319105607Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12c7da4c: reopening flushed file at 1732319105610 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@170f1b3f: reopening flushed file at 1732319105613 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@235daaed: reopening flushed file at 1732319105616 (+3 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 96ms, sequenceid=84, compaction requested=false at 1732319105619 (+3 ms)Writing region close event to WAL at 1732319105621 (+2 ms)Closed at 1732319105621 2024-11-22T23:45:05,623 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:45:05,623 INFO [M:0;a511e45b3024:41027 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T23:45:05,623 INFO [M:0;a511e45b3024:41027 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41027 2024-11-22T23:45:05,624 INFO [M:0;a511e45b3024:41027 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:45:05,726 INFO [M:0;a511e45b3024:41027 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:45:05,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:05,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41027-0x1014211a5a70000, quorum=127.0.0.1:50231, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:05,731 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T23:45:05,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T23:45:05,751 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:05,760 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/a511e45b3024:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732319103681 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a511e45b3024:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732319105379 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732319105375 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/a511e45b3024:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732319105206 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a511e45b3024:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 235) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9293 (was 9634) 2024-11-22T23:45:05,766 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=9293 2024-11-22T23:45:05,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/hadoop-log-dir so I do NOT create it in target/test-data/0e39ccda-0056-c991-21ee-e2098c595347 2024-11-22T23:45:05,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/hadoop-tmp-dir so I do NOT create it in target/test-data/0e39ccda-0056-c991-21ee-e2098c595347 2024-11-22T23:45:05,767 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:05,770 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/cluster_54ab8cf1-27cf-9f0b-8d8d-697d84651cf2, deleteOnExit=true 2024-11-22T23:45:05,776 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/cluster_54ab8cf1-27cf-9f0b-8d8d-697d84651cf2/zookeeper_0, clientPort=55949, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/cluster_54ab8cf1-27cf-9f0b-8d8d-697d84651cf2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/cluster_54ab8cf1-27cf-9f0b-8d8d-697d84651cf2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T23:45:05,777 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55949 2024-11-22T23:45:05,789 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 with version=8 2024-11-22T23:45:05,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/hbase-staging 2024-11-22T23:45:05,792 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:45:05,792 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:05,792 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:05,792 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:45:05,793 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:05,793 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:45:05,793 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T23:45:05,793 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:45:05,793 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40019 2024-11-22T23:45:05,795 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40019 connecting to ZooKeeper ensemble=127.0.0.1:55949 2024-11-22T23:45:05,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400190x0, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:45:05,805 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40019-0x1014211bd7d0000 connected 2024-11-22T23:45:05,822 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:05,822 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:05,823 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:05,823 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532, hbase.cluster.distributed=false 2024-11-22T23:45:05,826 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:45:05,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40019 2024-11-22T23:45:05,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40019 2024-11-22T23:45:05,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40019 2024-11-22T23:45:05,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40019 2024-11-22T23:45:05,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40019 2024-11-22T23:45:05,845 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:45:05,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:05,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:05,845 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:45:05,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:05,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:45:05,845 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T23:45:05,845 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:45:05,846 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34825 2024-11-22T23:45:05,848 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34825 connecting to ZooKeeper ensemble=127.0.0.1:55949 2024-11-22T23:45:05,848 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:05,848 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:05,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348250x0, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:45:05,853 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34825-0x1014211bd7d0001 connected 2024-11-22T23:45:05,853 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:05,854 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T23:45:05,854 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.cache 2024-11-22T23:45:05,855 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T23:45:05,855 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T23:45:05,855 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T23:45:05,855 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T23:45:05,855 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T23:45:05,855 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T23:45:05,856 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T23:45:05,856 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T23:45:05,856 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.persistence is complete. 2024-11-22T23:45:05,857 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T23:45:05,858 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:45:05,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34825 2024-11-22T23:45:05,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34825 2024-11-22T23:45:05,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34825 2024-11-22T23:45:05,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34825 2024-11-22T23:45:05,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34825 2024-11-22T23:45:05,875 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:45:05,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:05,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:05,876 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:45:05,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:05,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:45:05,876 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T23:45:05,876 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:45:05,877 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46175 2024-11-22T23:45:05,878 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46175 connecting to ZooKeeper ensemble=127.0.0.1:55949 2024-11-22T23:45:05,879 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:05,879 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:05,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461750x0, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:45:05,883 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46175-0x1014211bd7d0002 connected 2024-11-22T23:45:05,883 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:05,883 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T23:45:05,884 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.cache 2024-11-22T23:45:05,884 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T23:45:05,884 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T23:45:05,884 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T23:45:05,886 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T23:45:05,886 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T23:45:05,886 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T23:45:05,886 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T23:45:05,886 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.persistence is complete. 2024-11-22T23:45:05,887 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T23:45:05,888 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T23:45:05,889 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:45:05,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46175 2024-11-22T23:45:05,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46175 2024-11-22T23:45:05,892 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46175 2024-11-22T23:45:05,895 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46175 2024-11-22T23:45:05,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46175 2024-11-22T23:45:05,916 DEBUG [M:0;a511e45b3024:40019 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a511e45b3024:40019 2024-11-22T23:45:05,917 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a511e45b3024,40019,1732319105791 2024-11-22T23:45:05,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:05,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:05,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:05,920 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a511e45b3024,40019,1732319105791 2024-11-22T23:45:05,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T23:45:05,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T23:45:05,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:05,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:05,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:05,922 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T23:45:05,923 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a511e45b3024,40019,1732319105791 from backup master directory 2024-11-22T23:45:05,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a511e45b3024,40019,1732319105791 2024-11-22T23:45:05,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:05,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:05,926 WARN [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:45:05,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:45:05,926 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a511e45b3024,40019,1732319105791 2024-11-22T23:45:05,927 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/hbase.id] with ID: 3237467e-8543-4f49-90cf-ae1c913e75ce 2024-11-22T23:45:05,927 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/.tmp/hbase.id 2024-11-22T23:45:05,932 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/hbase.id] 2024-11-22T23:45:05,971 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:05,971 INFO [master/a511e45b3024:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T23:45:05,973 INFO [master/a511e45b3024:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-22T23:45:05,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:05,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:05,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:05,981 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:45:05,982 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T23:45:05,982 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:45:06,002 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store 2024-11-22T23:45:06,009 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:06,009 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:45:06,009 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:06,009 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:06,009 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:45:06,009 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:06,009 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:06,009 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732319106009Disabling compacts and flushes for region at 1732319106009Disabling writes for close at 1732319106009Writing region close event to WAL at 1732319106009Closed at 1732319106009 2024-11-22T23:45:06,009 WARN [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/.initializing 2024-11-22T23:45:06,009 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/WALs/a511e45b3024,40019,1732319105791 2024-11-22T23:45:06,015 INFO [master/a511e45b3024:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:45:06,018 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a511e45b3024%2C40019%2C1732319105791, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/WALs/a511e45b3024,40019,1732319105791, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/oldWALs, maxLogs=10 2024-11-22T23:45:06,037 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/WALs/a511e45b3024,40019,1732319105791/a511e45b3024%2C40019%2C1732319105791.1732319106019 2024-11-22T23:45:06,037 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:45:06,037 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:45:06,038 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:06,038 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,038 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T23:45:06,044 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:06,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T23:45:06,048 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:06,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T23:45:06,052 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:06,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T23:45:06,056 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:06,056 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,057 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,057 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,057 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,057 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,058 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:45:06,058 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:45:06,064 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:45:06,065 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62528003, jitterRate=-0.06826014816761017}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:45:06,066 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732319106038Initializing all the Stores at 1732319106038Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319106038Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319106038Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319106038Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319106039 (+1 ms)Cleaning up temporary data from old regions at 1732319106057 (+18 ms)Region opened successfully at 1732319106066 (+9 ms) 2024-11-22T23:45:06,066 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T23:45:06,070 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@222ea1ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:45:06,072 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T23:45:06,072 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T23:45:06,072 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T23:45:06,072 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:45:06,072 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:45:06,073 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-22T23:45:06,073 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T23:45:06,075 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T23:45:06,076 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T23:45:06,077 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T23:45:06,077 INFO [master/a511e45b3024:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T23:45:06,078 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T23:45:06,080 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T23:45:06,080 INFO [master/a511e45b3024:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T23:45:06,081 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T23:45:06,083 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T23:45:06,084 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T23:45:06,085 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T23:45:06,087 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T23:45:06,088 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T23:45:06,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:06,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:06,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:06,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,091 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a511e45b3024,40019,1732319105791, sessionid=0x1014211bd7d0000, setting cluster-up flag (Was=false) 2024-11-22T23:45:06,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,102 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T23:45:06,103 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a511e45b3024,40019,1732319105791 2024-11-22T23:45:06,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,113 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T23:45:06,114 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a511e45b3024,40019,1732319105791 2024-11-22T23:45:06,115 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T23:45:06,117 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:45:06,117 INFO [master/a511e45b3024:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T23:45:06,118 INFO [master/a511e45b3024:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T23:45:06,118 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a511e45b3024,40019,1732319105791 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T23:45:06,120 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:45:06,120 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:45:06,120 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:45:06,121 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:45:06,121 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a511e45b3024:0, corePoolSize=10, maxPoolSize=10 2024-11-22T23:45:06,121 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,121 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:45:06,121 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,122 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319136122 2024-11-22T23:45:06,122 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T23:45:06,122 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T23:45:06,122 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T23:45:06,122 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T23:45:06,122 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T23:45:06,122 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T23:45:06,122 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:45:06,123 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T23:45:06,123 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,123 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T23:45:06,123 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T23:45:06,123 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,123 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T23:45:06,123 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T23:45:06,124 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T23:45:06,124 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T23:45:06,124 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732319106124,5,FailOnTimeoutGroup] 2024-11-22T23:45:06,125 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732319106124,5,FailOnTimeoutGroup] 2024-11-22T23:45:06,125 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,125 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T23:45:06,125 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,125 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,140 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T23:45:06,140 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 2024-11-22T23:45:06,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:06,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T23:45:06,152 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:45:06,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:06,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T23:45:06,156 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:45:06,156 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:06,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T23:45:06,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:45:06,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:06,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T23:45:06,164 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:45:06,164 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:06,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:45:06,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740 2024-11-22T23:45:06,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740 2024-11-22T23:45:06,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:45:06,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:45:06,165 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:45:06,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:45:06,172 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:45:06,172 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64922722, jitterRate=-0.0325760543346405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:45:06,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732319106147Initializing all the Stores at 1732319106148 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319106148Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319106148Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319106148Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319106148Cleaning up temporary data from old regions at 1732319106165 (+17 ms)Region opened successfully at 1732319106173 (+8 ms) 2024-11-22T23:45:06,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:45:06,173 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:45:06,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:45:06,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:45:06,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:45:06,174 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:45:06,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732319106173Disabling compacts and flushes for region at 1732319106173Disabling writes for close at 1732319106173Writing region close event to WAL at 1732319106174 (+1 ms)Closed at 1732319106174 2024-11-22T23:45:06,175 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:45:06,175 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T23:45:06,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T23:45:06,176 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:45:06,177 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T23:45:06,199 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(746): ClusterId : 3237467e-8543-4f49-90cf-ae1c913e75ce 2024-11-22T23:45:06,199 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(746): ClusterId : 3237467e-8543-4f49-90cf-ae1c913e75ce 2024-11-22T23:45:06,199 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T23:45:06,199 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T23:45:06,202 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T23:45:06,202 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T23:45:06,202 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T23:45:06,202 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T23:45:06,204 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T23:45:06,204 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T23:45:06,205 DEBUG [RS:1;a511e45b3024:46175 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a3763ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:45:06,205 DEBUG [RS:0;a511e45b3024:34825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69d5d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:45:06,217 DEBUG [RS:1;a511e45b3024:46175 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a511e45b3024:46175 2024-11-22T23:45:06,217 INFO [RS:1;a511e45b3024:46175 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T23:45:06,217 INFO [RS:1;a511e45b3024:46175 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T23:45:06,217 DEBUG [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T23:45:06,218 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(2659): reportForDuty to master=a511e45b3024,40019,1732319105791 with port=46175, startcode=1732319105875 2024-11-22T23:45:06,219 DEBUG [RS:1;a511e45b3024:46175 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T23:45:06,221 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40073, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T23:45:06,222 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a511e45b3024,46175,1732319105875 2024-11-22T23:45:06,222 DEBUG [RS:0;a511e45b3024:34825 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a511e45b3024:34825 2024-11-22T23:45:06,222 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40019 {}] master.ServerManager(517): Registering regionserver=a511e45b3024,46175,1732319105875 2024-11-22T23:45:06,222 INFO [RS:0;a511e45b3024:34825 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T23:45:06,222 INFO [RS:0;a511e45b3024:34825 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T23:45:06,222 DEBUG [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T23:45:06,223 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(2659): reportForDuty to master=a511e45b3024,40019,1732319105791 with port=34825, startcode=1732319105844 2024-11-22T23:45:06,224 DEBUG [RS:0;a511e45b3024:34825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T23:45:06,224 DEBUG [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 2024-11-22T23:45:06,224 DEBUG [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T23:45:06,224 DEBUG [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T23:45:06,226 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41569, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T23:45:06,227 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a511e45b3024,34825,1732319105844 2024-11-22T23:45:06,227 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40019 {}] master.ServerManager(517): Registering regionserver=a511e45b3024,34825,1732319105844 2024-11-22T23:45:06,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:45:06,228 DEBUG [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 2024-11-22T23:45:06,228 DEBUG [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T23:45:06,228 DEBUG [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T23:45:06,228 DEBUG [RS:1;a511e45b3024:46175 {}] zookeeper.ZKUtil(111): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a511e45b3024,46175,1732319105875 2024-11-22T23:45:06,229 WARN [RS:1;a511e45b3024:46175 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:45:06,229 INFO [RS:1;a511e45b3024:46175 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:45:06,229 DEBUG [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,46175,1732319105875 2024-11-22T23:45:06,229 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a511e45b3024,46175,1732319105875] 2024-11-22T23:45:06,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:45:06,234 DEBUG [RS:0;a511e45b3024:34825 {}] zookeeper.ZKUtil(111): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a511e45b3024,34825,1732319105844 2024-11-22T23:45:06,234 WARN [RS:0;a511e45b3024:34825 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:45:06,234 INFO [RS:0;a511e45b3024:34825 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:45:06,234 DEBUG [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34825,1732319105844 2024-11-22T23:45:06,234 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a511e45b3024,34825,1732319105844] 2024-11-22T23:45:06,236 INFO [RS:1;a511e45b3024:46175 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T23:45:06,238 INFO [RS:1;a511e45b3024:46175 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T23:45:06,239 INFO [RS:0;a511e45b3024:34825 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T23:45:06,239 INFO [RS:1;a511e45b3024:46175 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T23:45:06,239 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,239 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T23:45:06,240 INFO [RS:1;a511e45b3024:46175 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T23:45:06,240 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,240 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,240 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,240 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,241 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,241 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,241 INFO [RS:0;a511e45b3024:34825 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T23:45:06,241 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:45:06,241 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,241 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,241 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,241 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,241 INFO [RS:0;a511e45b3024:34825 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T23:45:06,241 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,241 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,241 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,241 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:45:06,241 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T23:45:06,242 DEBUG [RS:1;a511e45b3024:46175 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:45:06,242 INFO [RS:0;a511e45b3024:34825 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T23:45:06,242 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,242 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,242 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,242 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,242 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,242 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,242 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,242 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,242 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,242 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,242 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,46175,1732319105875-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:45:06,243 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,243 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:45:06,243 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,243 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,243 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,243 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,243 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,243 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:06,243 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:45:06,243 DEBUG [RS:0;a511e45b3024:34825 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:45:06,248 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,248 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,248 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,249 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,249 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,249 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,34825,1732319105844-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:45:06,259 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T23:45:06,259 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,46175,1732319105875-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,259 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,260 INFO [RS:1;a511e45b3024:46175 {}] regionserver.Replication(171): a511e45b3024,46175,1732319105875 started 2024-11-22T23:45:06,263 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T23:45:06,264 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,34825,1732319105844-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,264 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,264 INFO [RS:0;a511e45b3024:34825 {}] regionserver.Replication(171): a511e45b3024,34825,1732319105844 started 2024-11-22T23:45:06,275 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,275 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(1482): Serving as a511e45b3024,46175,1732319105875, RpcServer on a511e45b3024/172.17.0.2:46175, sessionid=0x1014211bd7d0002 2024-11-22T23:45:06,276 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T23:45:06,276 DEBUG [RS:1;a511e45b3024:46175 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a511e45b3024,46175,1732319105875 2024-11-22T23:45:06,276 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,46175,1732319105875' 2024-11-22T23:45:06,276 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T23:45:06,277 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T23:45:06,277 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T23:45:06,277 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T23:45:06,277 DEBUG [RS:1;a511e45b3024:46175 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a511e45b3024,46175,1732319105875 2024-11-22T23:45:06,277 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,46175,1732319105875' 2024-11-22T23:45:06,277 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T23:45:06,278 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T23:45:06,278 DEBUG [RS:1;a511e45b3024:46175 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T23:45:06,278 INFO [RS:1;a511e45b3024:46175 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T23:45:06,278 INFO [RS:1;a511e45b3024:46175 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T23:45:06,279 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,279 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(1482): Serving as a511e45b3024,34825,1732319105844, RpcServer on a511e45b3024/172.17.0.2:34825, sessionid=0x1014211bd7d0001 2024-11-22T23:45:06,279 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T23:45:06,279 DEBUG [RS:0;a511e45b3024:34825 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a511e45b3024,34825,1732319105844 2024-11-22T23:45:06,279 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,34825,1732319105844' 2024-11-22T23:45:06,279 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T23:45:06,279 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T23:45:06,280 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T23:45:06,280 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T23:45:06,280 DEBUG [RS:0;a511e45b3024:34825 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a511e45b3024,34825,1732319105844 2024-11-22T23:45:06,280 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,34825,1732319105844' 2024-11-22T23:45:06,280 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T23:45:06,280 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T23:45:06,281 DEBUG [RS:0;a511e45b3024:34825 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T23:45:06,281 INFO [RS:0;a511e45b3024:34825 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T23:45:06,281 INFO [RS:0;a511e45b3024:34825 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T23:45:06,327 WARN [a511e45b3024:40019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T23:45:06,379 INFO [RS:1;a511e45b3024:46175 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:45:06,380 INFO [RS:1;a511e45b3024:46175 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a511e45b3024%2C46175%2C1732319105875, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,46175,1732319105875, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs, maxLogs=55 2024-11-22T23:45:06,381 INFO [RS:0;a511e45b3024:34825 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:45:06,382 INFO [RS:0;a511e45b3024:34825 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a511e45b3024%2C34825%2C1732319105844, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34825,1732319105844, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs, maxLogs=55 2024-11-22T23:45:06,399 INFO [RS:1;a511e45b3024:46175 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,46175,1732319105875/a511e45b3024%2C46175%2C1732319105875.1732319106381 2024-11-22T23:45:06,400 DEBUG [RS:1;a511e45b3024:46175 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:45:06,400 INFO [RS:0;a511e45b3024:34825 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34825,1732319105844/a511e45b3024%2C34825%2C1732319105844.1732319106383 2024-11-22T23:45:06,400 DEBUG [RS:0;a511e45b3024:34825 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:45:06,578 DEBUG [a511e45b3024:40019 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-22T23:45:06,578 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(204): Hosts are {a511e45b3024=0} racks are {/default-rack=0} 2024-11-22T23:45:06,579 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T23:45:06,579 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T23:45:06,579 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T23:45:06,579 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T23:45:06,579 INFO [a511e45b3024:40019 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T23:45:06,579 INFO [a511e45b3024:40019 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T23:45:06,579 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T23:45:06,579 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a511e45b3024,34825,1732319105844 2024-11-22T23:45:06,580 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,34825,1732319105844, state=OPENING 2024-11-22T23:45:06,582 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T23:45:06,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:06,584 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:45:06,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:06,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a511e45b3024,34825,1732319105844}] 2024-11-22T23:45:06,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:06,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:06,737 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T23:45:06,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60369, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T23:45:06,742 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T23:45:06,742 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:45:06,743 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T23:45:06,744 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a511e45b3024%2C34825%2C1732319105844.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34825,1732319105844, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs, maxLogs=55 2024-11-22T23:45:06,772 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34825,1732319105844/a511e45b3024%2C34825%2C1732319105844.meta.1732319106745.meta 2024-11-22T23:45:06,773 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:45:06,773 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:45:06,773 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T23:45:06,774 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T23:45:06,774 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T23:45:06,774 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T23:45:06,774 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:06,774 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T23:45:06,774 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T23:45:06,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T23:45:06,776 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:45:06,776 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:06,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T23:45:06,777 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:45:06,777 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:06,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T23:45:06,778 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:45:06,778 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:06,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T23:45:06,778 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:45:06,778 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:06,779 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:45:06,779 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740 2024-11-22T23:45:06,780 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740 2024-11-22T23:45:06,780 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:45:06,780 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:45:06,780 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:45:06,781 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:45:06,781 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69483647, jitterRate=0.03538702428340912}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:45:06,781 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T23:45:06,782 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732319106774Writing region info on filesystem at 1732319106774Initializing all the Stores at 1732319106775 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319106775Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319106775Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319106775Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319106775Cleaning up temporary data from old regions at 1732319106780 (+5 ms)Running coprocessor post-open hooks at 1732319106781 (+1 ms)Region opened successfully at 1732319106782 (+1 ms) 2024-11-22T23:45:06,783 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732319106737 2024-11-22T23:45:06,786 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T23:45:06,786 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T23:45:06,786 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,34825,1732319105844 2024-11-22T23:45:06,787 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,34825,1732319105844, state=OPEN 2024-11-22T23:45:06,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:06,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:06,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:06,790 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:06,790 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:06,790 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:06,790 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a511e45b3024,34825,1732319105844 2024-11-22T23:45:06,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T23:45:06,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a511e45b3024,34825,1732319105844 in 206 msec 2024-11-22T23:45:06,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T23:45:06,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 617 msec 2024-11-22T23:45:06,794 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:45:06,794 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T23:45:06,798 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:45:06,798 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,34825,1732319105844, seqNum=-1] 2024-11-22T23:45:06,799 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:45:06,801 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:45:06,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 688 msec 2024-11-22T23:45:06,806 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732319106806, completionTime=-1 2024-11-22T23:45:06,806 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-22T23:45:06,806 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:45:06,809 INFO [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-22T23:45:06,809 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319166809 2024-11-22T23:45:06,809 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319226809 2024-11-22T23:45:06,809 INFO [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-22T23:45:06,809 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,40019,1732319105791-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,810 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,40019,1732319105791-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,810 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,40019,1732319105791-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,810 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a511e45b3024:40019, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,810 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,810 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,813 DEBUG [master/a511e45b3024:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T23:45:06,815 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.889sec 2024-11-22T23:45:06,816 INFO [master/a511e45b3024:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T23:45:06,816 INFO [master/a511e45b3024:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T23:45:06,816 INFO [master/a511e45b3024:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T23:45:06,816 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T23:45:06,816 INFO [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T23:45:06,816 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,40019,1732319105791-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:45:06,816 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,40019,1732319105791-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T23:45:06,817 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T23:45:06,817 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T23:45:06,817 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,40019,1732319105791-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:06,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:45:06,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,40019,-1 for getting cluster id 2024-11-22T23:45:06,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:45:06,901 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3237467e-8543-4f49-90cf-ae1c913e75ce' 2024-11-22T23:45:06,902 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:45:06,902 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3237467e-8543-4f49-90cf-ae1c913e75ce" 2024-11-22T23:45:06,902 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a88e976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:45:06,903 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,40019,-1] 2024-11-22T23:45:06,903 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:45:06,903 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:06,905 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:45:06,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af0bbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:45:06,907 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:45:06,908 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,34825,1732319105844, seqNum=-1] 2024-11-22T23:45:06,909 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:45:06,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:45:06,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a511e45b3024,40019,1732319105791 2024-11-22T23:45:06,917 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:45:06,918 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is a511e45b3024,40019,1732319105791 2024-11-22T23:45:06,918 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@194bcc11 2024-11-22T23:45:06,918 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:45:06,921 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:45:06,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732319106916', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:45:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732319106916 2024-11-22T23:45:06,924 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732319106916 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:45:06,924 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:06,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732319106916" procId is: 4 2024-11-22T23:45:06,925 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732319106916 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:45:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:45:06,935 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732319106916-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aa4b9fab2da1a9ae656805c6c4ea954b, NAME => 'testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732319106916', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 2024-11-22T23:45:06,944 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732319106916-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:06,945 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732319106916-pool-0 {}] regionserver.HRegion(1722): Closing aa4b9fab2da1a9ae656805c6c4ea954b, disabling compactions & flushes 2024-11-22T23:45:06,945 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732319106916-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:06,945 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732319106916-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:06,945 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732319106916-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. after waiting 0 ms 2024-11-22T23:45:06,945 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732319106916-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:06,945 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732319106916-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:06,945 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732319106916-pool-0 {}] regionserver.HRegion(1676): Region close journal for aa4b9fab2da1a9ae656805c6c4ea954b: Waiting for close lock at 1732319106944Disabling compacts and flushes for region at 1732319106944Disabling writes for close at 1732319106945 (+1 ms)Writing region close event to WAL at 1732319106945Closed at 1732319106945 2024-11-22T23:45:06,946 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732319106916 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:45:06,946 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732319106946"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732319106946"}]},"ts":"1732319106946"} 2024-11-22T23:45:06,949 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:45:06,950 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732319106916 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:45:06,951 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732319106916","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319106950"}]},"ts":"1732319106950"} 2024-11-22T23:45:06,953 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732319106916, state=ENABLING in hbase:meta 2024-11-22T23:45:06,953 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a511e45b3024=0} racks are {/default-rack=0} 2024-11-22T23:45:06,954 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T23:45:06,954 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T23:45:06,954 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T23:45:06,954 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T23:45:06,954 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T23:45:06,954 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T23:45:06,954 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T23:45:06,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732319106916, region=aa4b9fab2da1a9ae656805c6c4ea954b, ASSIGN}] 2024-11-22T23:45:06,955 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732319106916, region=aa4b9fab2da1a9ae656805c6c4ea954b, ASSIGN 2024-11-22T23:45:06,956 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732319106916, region=aa4b9fab2da1a9ae656805c6c4ea954b, ASSIGN; state=OFFLINE, location=a511e45b3024,34825,1732319105844; forceNewPlan=false, retain=false 2024-11-22T23:45:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:45:07,107 INFO [a511e45b3024:40019 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-22T23:45:07,107 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=aa4b9fab2da1a9ae656805c6c4ea954b, regionState=OPENING, regionLocation=a511e45b3024,34825,1732319105844 2024-11-22T23:45:07,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732319106916, region=aa4b9fab2da1a9ae656805c6c4ea954b, ASSIGN because future has completed 2024-11-22T23:45:07,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa4b9fab2da1a9ae656805c6c4ea954b, server=a511e45b3024,34825,1732319105844}] 2024-11-22T23:45:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:45:07,265 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:07,266 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => aa4b9fab2da1a9ae656805c6c4ea954b, NAME => 'testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:45:07,266 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732319106916 aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,267 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:07,267 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,267 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,267 INFO [StoreOpener-aa4b9fab2da1a9ae656805c6c4ea954b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,272 INFO [StoreOpener-aa4b9fab2da1a9ae656805c6c4ea954b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa4b9fab2da1a9ae656805c6c4ea954b columnFamilyName family 2024-11-22T23:45:07,272 DEBUG [StoreOpener-aa4b9fab2da1a9ae656805c6c4ea954b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:07,272 INFO [StoreOpener-aa4b9fab2da1a9ae656805c6c4ea954b-1 {}] regionserver.HStore(327): Store=aa4b9fab2da1a9ae656805c6c4ea954b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:07,272 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,273 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,273 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,273 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,273 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,274 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,281 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:45:07,282 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened aa4b9fab2da1a9ae656805c6c4ea954b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73643187, jitterRate=0.0973690003156662}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T23:45:07,282 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:07,283 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for aa4b9fab2da1a9ae656805c6c4ea954b: Running coprocessor pre-open hook at 1732319107267Writing region info on filesystem at 1732319107267Initializing all the Stores at 1732319107267Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319107267Cleaning up temporary data from old regions at 1732319107273 (+6 ms)Running coprocessor post-open hooks at 1732319107282 (+9 ms)Region opened successfully at 1732319107283 (+1 ms) 2024-11-22T23:45:07,284 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b., pid=6, masterSystemTime=1732319107263 2024-11-22T23:45:07,286 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:07,287 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:07,287 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=aa4b9fab2da1a9ae656805c6c4ea954b, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,34825,1732319105844 2024-11-22T23:45:07,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa4b9fab2da1a9ae656805c6c4ea954b, server=a511e45b3024,34825,1732319105844 because future has completed 2024-11-22T23:45:07,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T23:45:07,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure aa4b9fab2da1a9ae656805c6c4ea954b, server=a511e45b3024,34825,1732319105844 in 180 msec 2024-11-22T23:45:07,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T23:45:07,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732319106916, region=aa4b9fab2da1a9ae656805c6c4ea954b, ASSIGN in 339 msec 2024-11-22T23:45:07,295 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732319106916 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:45:07,296 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732319106916","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319107295"}]},"ts":"1732319107295"} 2024-11-22T23:45:07,298 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732319106916, state=ENABLED in hbase:meta 2024-11-22T23:45:07,299 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732319106916 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:45:07,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732319106916 in 377 msec 2024-11-22T23:45:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:45:07,554 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732319106916 completed 2024-11-22T23:45:07,554 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732319106916 get assigned. Timeout = 60000ms 2024-11-22T23:45:07,554 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T23:45:07,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732319106916 assigned to meta. Checking AM states. 2024-11-22T23:45:07,558 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T23:45:07,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732319106916 assigned. 2024-11-22T23:45:07,562 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732319106916', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b., hostname=a511e45b3024,34825,1732319105844, seqNum=2] 2024-11-22T23:45:07,566 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing aa4b9fab2da1a9ae656805c6c4ea954b 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T23:45:07,577 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ed2631e40f6e4b8ebc527582241ec61b 2024-11-22T23:45:07,591 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/.tmp/family/ed2631e40f6e4b8ebc527582241ec61b is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T23:45:07,592 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/.tmp/family/ed2631e40f6e4b8ebc527582241ec61b 2024-11-22T23:45:07,595 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/.tmp/family/ed2631e40f6e4b8ebc527582241ec61b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b 2024-11-22T23:45:07,596 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b, delay=946 ms 2024-11-22T23:45:07,597 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T23:45:07,598 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for aa4b9fab2da1a9ae656805c6c4ea954b in 32ms, sequenceid=6, compaction requested=false 2024-11-22T23:45:07,598 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732319106916' 2024-11-22T23:45:07,599 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for aa4b9fab2da1a9ae656805c6c4ea954b: 2024-11-22T23:45:08,093 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:45:08,543 DEBUG [hfile-prefetch-1732319105206 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ed2631e40f6e4b8ebc527582241ec61b_0. 2024-11-22T23:45:08,544 DEBUG [hfile-prefetch-1732319105206 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ed2631e40f6e4b8ebc527582241ec61b_117. 2024-11-22T23:45:08,544 DEBUG [hfile-prefetch-1732319105206 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b, with total blocks 2, and data blocks 1 2024-11-22T23:45:08,544 DEBUG [hfile-prefetch-1732319105206 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T23:45:08,544 DEBUG [hfile-prefetch-1732319105206 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T23:45:08,544 DEBUG [hfile-prefetch-1732319105206 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b. Will sleep 100ms and try the verification again. 2024-11-22T23:45:08,557 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T23:45:08,557 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T23:45:08,557 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T23:45:08,557 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T23:45:08,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T23:45:08,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732319108557 2024-11-22T23:45:08,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T23:45:08,599 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;a511e45b3024:34825,5,FailOnTimeoutGroup] 2024-11-22T23:45:08,599 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a511e45b3024,34825,1732319105844' ***** 2024-11-22T23:45:08,599 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-22T23:45:08,599 INFO [RS:0;a511e45b3024:34825 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T23:45:08,600 INFO [RS:0;a511e45b3024:34825 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.persistence 2024-11-22T23:45:08,600 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T23:45:08,600 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T23:45:08,600 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T23:45:08,600 INFO [RS:0;a511e45b3024:34825 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T23:45:08,600 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T23:45:08,610 DEBUG [RS:0;a511e45b3024:34825 {}] bucket.BucketCache(1487): Thread RS:0;a511e45b3024:34825 started persisting bucket cache to file 2024-11-22T23:45:08,610 DEBUG [RS:0;a511e45b3024:34825 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T23:45:08,610 DEBUG [RS:0;a511e45b3024:34825 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T23:45:08,614 DEBUG [RS:0;a511e45b3024:34825 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T23:45:08,614 DEBUG [RS:0;a511e45b3024:34825 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732319108610 2024-11-22T23:45:08,614 DEBUG [RS:0;a511e45b3024:34825 {}] bucket.BucketCache(1509): Thread RS:0;a511e45b3024:34825 finished persisting bucket cache to file, renaming 2024-11-22T23:45:08,614 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T23:45:08,614 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T23:45:08,614 INFO [RS:0;a511e45b3024:34825 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T23:45:08,614 INFO [RS:0;a511e45b3024:34825 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T23:45:08,614 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(3091): Received CLOSE for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:08,615 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(959): stopping server a511e45b3024,34825,1732319105844 2024-11-22T23:45:08,615 INFO [RS:0;a511e45b3024:34825 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:45:08,615 INFO [RS:0;a511e45b3024:34825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a511e45b3024:34825. 2024-11-22T23:45:08,615 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T23:45:08,615 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aa4b9fab2da1a9ae656805c6c4ea954b, disabling compactions & flushes 2024-11-22T23:45:08,615 DEBUG [RS:0;a511e45b3024:34825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:45:08,615 DEBUG [RS:0;a511e45b3024:34825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:08,615 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:08,615 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:08,615 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. after waiting 0 ms 2024-11-22T23:45:08,615 INFO [RS:0;a511e45b3024:34825 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T23:45:08,615 INFO [RS:0;a511e45b3024:34825 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T23:45:08,615 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:08,615 INFO [RS:0;a511e45b3024:34825 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T23:45:08,615 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T23:45:08,616 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-22T23:45:08,616 DEBUG [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, aa4b9fab2da1a9ae656805c6c4ea954b=testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b.} 2024-11-22T23:45:08,616 DEBUG [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:08,616 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T23:45:08,616 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:45:08,616 DEBUG [StoreFileCloser-aa4b9fab2da1a9ae656805c6c4ea954b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b 2024-11-22T23:45:08,616 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:45:08,616 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:45:08,616 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:45:08,616 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:45:08,616 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-22T23:45:08,618 WARN [hfile-prefetch-1732319105206 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-22T23:45:08,619 DEBUG [hfile-prefetch-1732319105206 {}] hfile.PrefetchExecutor(121): Prefetch completed for ed2631e40f6e4b8ebc527582241ec61b 2024-11-22T23:45:08,622 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T23:45:08,622 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:08,622 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aa4b9fab2da1a9ae656805c6c4ea954b: Waiting for close lock at 1732319108615Running coprocessor pre-close hooks at 1732319108615Disabling compacts and flushes for region at 1732319108615Disabling writes for close at 1732319108615Writing region close event to WAL at 1732319108616 (+1 ms)Running coprocessor post-close hooks at 1732319108622 (+6 ms)Closed at 1732319108622 2024-11-22T23:45:08,622 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:08,627 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b8e03362d4ea4663867d220e66115796 2024-11-22T23:45:08,640 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/info/b8e03362d4ea4663867d220e66115796 is 209, key is testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b./info:regioninfo/1732319107287/Put/seqid=0 2024-11-22T23:45:08,641 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/info/b8e03362d4ea4663867d220e66115796 2024-11-22T23:45:08,649 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bda423a0c6ec48239ce7daad7214f599 2024-11-22T23:45:08,651 INFO [regionserver/a511e45b3024:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:45:08,662 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/ns/bda423a0c6ec48239ce7daad7214f599 is 43, key is default/ns:d/1732319106801/Put/seqid=0 2024-11-22T23:45:08,663 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/ns/bda423a0c6ec48239ce7daad7214f599 2024-11-22T23:45:08,671 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cf8923cd3a6a4395ad911b7927d9951d 2024-11-22T23:45:08,683 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/table/cf8923cd3a6a4395ad911b7927d9951d is 80, key is testBlockEvictionOnGracefulStop1732319106916/table:state/1732319107295/Put/seqid=0 2024-11-22T23:45:08,684 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/table/cf8923cd3a6a4395ad911b7927d9951d 2024-11-22T23:45:08,687 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/info/b8e03362d4ea4663867d220e66115796 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/b8e03362d4ea4663867d220e66115796 2024-11-22T23:45:08,688 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/b8e03362d4ea4663867d220e66115796, delay=933 ms 2024-11-22T23:45:08,689 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/b8e03362d4ea4663867d220e66115796, entries=10, sequenceid=11, filesize=7.0 K 2024-11-22T23:45:08,690 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/ns/bda423a0c6ec48239ce7daad7214f599 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/bda423a0c6ec48239ce7daad7214f599 2024-11-22T23:45:08,691 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/bda423a0c6ec48239ce7daad7214f599, delay=966 ms 2024-11-22T23:45:08,692 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/bda423a0c6ec48239ce7daad7214f599, entries=2, sequenceid=11, filesize=5.0 K 2024-11-22T23:45:08,692 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/table/cf8923cd3a6a4395ad911b7927d9951d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/cf8923cd3a6a4395ad911b7927d9951d 2024-11-22T23:45:08,693 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/cf8923cd3a6a4395ad911b7927d9951d, delay=948 ms 2024-11-22T23:45:08,694 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/cf8923cd3a6a4395ad911b7927d9951d, entries=2, sequenceid=11, filesize=5.3 K 2024-11-22T23:45:08,695 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=11, compaction requested=false 2024-11-22T23:45:08,695 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/b8e03362d4ea4663867d220e66115796 2024-11-22T23:45:08,696 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/bda423a0c6ec48239ce7daad7214f599 2024-11-22T23:45:08,696 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/cf8923cd3a6a4395ad911b7927d9951d 2024-11-22T23:45:08,701 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-22T23:45:08,701 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T23:45:08,701 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:45:08,701 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732319108616Running coprocessor pre-close hooks at 1732319108616Disabling compacts and flushes for region at 1732319108616Disabling writes for close at 1732319108616Obtaining lock to block concurrent updates at 1732319108616Preparing flush snapshotting stores in 1588230740 at 1732319108616Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732319108617 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732319108617Flushing 1588230740/info: creating writer at 1732319108617Flushing 1588230740/info: appending metadata at 1732319108640 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732319108640Flushing 1588230740/ns: creating writer at 1732319108643 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732319108661 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732319108661Flushing 1588230740/table: creating writer at 1732319108665 (+4 ms)Flushing 1588230740/table: appending metadata at 1732319108683 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732319108683Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36f81c95: reopening flushed file at 1732319108686 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@326e0457: reopening flushed file at 1732319108689 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@291c66f5: reopening flushed file at 1732319108692 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=11, compaction requested=false at 1732319108695 (+3 ms)Writing region close event to WAL at 1732319108697 (+2 ms)Running coprocessor post-close hooks at 1732319108701 (+4 ms)Closed at 1732319108701 2024-11-22T23:45:08,701 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T23:45:08,816 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(976): stopping server a511e45b3024,34825,1732319105844; all regions closed. 2024-11-22T23:45:08,818 DEBUG [RS:0;a511e45b3024:34825 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs 2024-11-22T23:45:08,818 INFO [RS:0;a511e45b3024:34825 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C34825%2C1732319105844.meta:.meta(num 1732319106745) 2024-11-22T23:45:08,820 DEBUG [RS:0;a511e45b3024:34825 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs 2024-11-22T23:45:08,820 INFO [RS:0;a511e45b3024:34825 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C34825%2C1732319105844:(num 1732319106383) 2024-11-22T23:45:08,820 DEBUG [RS:0;a511e45b3024:34825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:08,820 INFO [RS:0;a511e45b3024:34825 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:45:08,820 INFO [RS:0;a511e45b3024:34825 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:45:08,820 INFO [RS:0;a511e45b3024:34825 {}] hbase.ChoreService(370): Chore service for: regionserver/a511e45b3024:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T23:45:08,820 INFO [RS:0;a511e45b3024:34825 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:45:08,820 INFO [regionserver/a511e45b3024:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:45:08,820 INFO [RS:0;a511e45b3024:34825 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34825 2024-11-22T23:45:08,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a511e45b3024,34825,1732319105844 2024-11-22T23:45:08,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:45:08,825 INFO [RS:0;a511e45b3024:34825 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:45:08,826 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a511e45b3024,34825,1732319105844] 2024-11-22T23:45:08,829 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a511e45b3024,34825,1732319105844 already deleted, retry=false 2024-11-22T23:45:08,829 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a511e45b3024,34825,1732319105844 on a511e45b3024,40019,1732319105791 2024-11-22T23:45:08,833 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a511e45b3024,34825,1732319105844, splitWal=true, meta=true 2024-11-22T23:45:08,836 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for a511e45b3024,34825,1732319105844 (carryingMeta=true) a511e45b3024,34825,1732319105844/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@282ffa9b[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-22T23:45:08,836 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a511e45b3024,34825,1732319105844, splitWal=true, meta=true 2024-11-22T23:45:08,837 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a511e45b3024,34825,1732319105844, splitWal=true, meta=true, isMeta: true 2024-11-22T23:45:08,837 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34825,1732319105844-splitting 2024-11-22T23:45:08,837 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34825,1732319105844-splitting dir is empty, no logs to split. 2024-11-22T23:45:08,838 INFO [PEWorker-2 {}] master.SplitWALManager(105): a511e45b3024,34825,1732319105844 WAL count=0, meta=true 2024-11-22T23:45:08,839 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34825,1732319105844-splitting dir is empty, no logs to split. 2024-11-22T23:45:08,839 INFO [PEWorker-2 {}] master.SplitWALManager(105): a511e45b3024,34825,1732319105844 WAL count=0, meta=true 2024-11-22T23:45:08,839 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if a511e45b3024,34825,1732319105844 WAL splitting is done? wals=0, meta=true 2024-11-22T23:45:08,841 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T23:45:08,842 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:45:08,843 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T23:45:08,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:08,927 INFO [RS:0;a511e45b3024:34825 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:45:08,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34825-0x1014211bd7d0001, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:08,927 INFO [RS:0;a511e45b3024:34825 {}] regionserver.HRegionServer(1031): Exiting; stopping=a511e45b3024,34825,1732319105844; zookeeper connection closed. 2024-11-22T23:45:08,927 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3487272e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3487272e 2024-11-22T23:45:08,993 DEBUG [a511e45b3024:40019 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T23:45:08,994 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a511e45b3024,46175,1732319105875 2024-11-22T23:45:08,994 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,46175,1732319105875, state=OPENING 2024-11-22T23:45:08,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:08,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:08,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:08,996 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:45:08,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:08,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a511e45b3024,46175,1732319105875}] 2024-11-22T23:45:09,117 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:45:09,117 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:09,117 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:09,117 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:45:09,117 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:45:09,117 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:45:09,117 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T23:45:09,118 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:45:09,118 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44219 2024-11-22T23:45:09,120 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44219 connecting to ZooKeeper ensemble=127.0.0.1:55949 2024-11-22T23:45:09,120 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:09,120 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:09,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442190x0, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:45:09,123 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44219-0x1014211bd7d0003 connected 2024-11-22T23:45:09,123 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-22T23:45:09,123 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-22T23:45:09,124 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T23:45:09,125 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.cache 2024-11-22T23:45:09,125 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T23:45:09,125 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T23:45:09,125 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-22T23:45:09,125 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T23:45:09,125 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T23:45:09,126 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T23:45:09,126 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T23:45:09,126 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-22T23:45:09,128 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-22T23:45:09,128 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:45:09,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44219 2024-11-22T23:45:09,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44219 2024-11-22T23:45:09,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44219 2024-11-22T23:45:09,133 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-22T23:45:09,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44219 2024-11-22T23:45:09,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44219 2024-11-22T23:45:09,134 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T23:45:09,134 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-22T23:45:09,134 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.persistence is complete. 2024-11-22T23:45:09,135 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(746): ClusterId : 3237467e-8543-4f49-90cf-ae1c913e75ce 2024-11-22T23:45:09,135 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T23:45:09,137 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T23:45:09,138 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T23:45:09,141 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T23:45:09,141 DEBUG [RS:2;a511e45b3024:44219 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58dcca52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:45:09,149 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T23:45:09,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60487, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T23:45:09,153 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T23:45:09,153 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:45:09,154 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T23:45:09,154 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a511e45b3024%2C46175%2C1732319105875.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,46175,1732319105875, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs, maxLogs=55 2024-11-22T23:45:09,155 DEBUG [RS:2;a511e45b3024:44219 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a511e45b3024:44219 2024-11-22T23:45:09,155 INFO [RS:2;a511e45b3024:44219 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T23:45:09,155 INFO [RS:2;a511e45b3024:44219 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T23:45:09,156 DEBUG [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T23:45:09,156 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(2659): reportForDuty to master=a511e45b3024,40019,1732319105791 with port=44219, startcode=1732319109117 2024-11-22T23:45:09,157 DEBUG [RS:2;a511e45b3024:44219 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T23:45:09,159 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54511, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T23:45:09,159 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a511e45b3024,44219,1732319109117 2024-11-22T23:45:09,159 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40019 {}] master.ServerManager(517): Registering regionserver=a511e45b3024,44219,1732319109117 2024-11-22T23:45:09,161 DEBUG [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532 2024-11-22T23:45:09,161 DEBUG [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T23:45:09,161 DEBUG [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T23:45:09,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:45:09,163 DEBUG [RS:2;a511e45b3024:44219 {}] zookeeper.ZKUtil(111): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a511e45b3024,44219,1732319109117 2024-11-22T23:45:09,163 WARN [RS:2;a511e45b3024:44219 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:45:09,163 INFO [RS:2;a511e45b3024:44219 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:45:09,163 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a511e45b3024,44219,1732319109117] 2024-11-22T23:45:09,164 DEBUG [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,44219,1732319109117 2024-11-22T23:45:09,169 INFO [RS:2;a511e45b3024:44219 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T23:45:09,172 INFO [RS:2;a511e45b3024:44219 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T23:45:09,173 INFO [RS:2;a511e45b3024:44219 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T23:45:09,173 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:09,173 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T23:45:09,173 INFO [RS:2;a511e45b3024:44219 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T23:45:09,173 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:45:09,174 DEBUG [RS:2;a511e45b3024:44219 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:45:09,176 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:09,176 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:09,177 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:09,177 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:09,177 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:09,177 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,44219,1732319109117-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:45:09,177 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,46175,1732319105875/a511e45b3024%2C46175%2C1732319105875.meta.1732319109155.meta 2024-11-22T23:45:09,179 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:45:09,179 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:45:09,179 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T23:45:09,180 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T23:45:09,180 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T23:45:09,180 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T23:45:09,180 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:09,180 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T23:45:09,180 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T23:45:09,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T23:45:09,182 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:45:09,182 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:09,185 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/b8e03362d4ea4663867d220e66115796, delay=985 ms 2024-11-22T23:45:09,186 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/b8e03362d4ea4663867d220e66115796 2024-11-22T23:45:09,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:09,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T23:45:09,187 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:45:09,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:09,189 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/bda423a0c6ec48239ce7daad7214f599, delay=908 ms 2024-11-22T23:45:09,190 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/bda423a0c6ec48239ce7daad7214f599 2024-11-22T23:45:09,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:09,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T23:45:09,191 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:45:09,191 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:09,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:09,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T23:45:09,192 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:45:09,192 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:09,192 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T23:45:09,192 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,44219,1732319109117-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:09,192 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:09,193 INFO [RS:2;a511e45b3024:44219 {}] regionserver.Replication(171): a511e45b3024,44219,1732319109117 started 2024-11-22T23:45:09,193 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/cf8923cd3a6a4395ad911b7927d9951d, delay=948 ms 2024-11-22T23:45:09,194 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/cf8923cd3a6a4395ad911b7927d9951d 2024-11-22T23:45:09,194 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:45:09,194 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:45:09,195 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740 2024-11-22T23:45:09,195 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740 2024-11-22T23:45:09,196 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:45:09,196 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:45:09,196 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:45:09,197 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:45:09,198 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68275758, jitterRate=0.017388075590133667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:45:09,198 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T23:45:09,199 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732319109180Writing region info on filesystem at 1732319109180Initializing all the Stores at 1732319109180Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319109180Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319109181 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319109181Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319109181Cleaning up temporary data from old regions at 1732319109196 (+15 ms)Running coprocessor post-open hooks at 1732319109198 (+2 ms)Region opened successfully at 1732319109199 (+1 ms) 2024-11-22T23:45:09,200 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732319109149 2024-11-22T23:45:09,202 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T23:45:09,202 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T23:45:09,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=a511e45b3024,46175,1732319105875 2024-11-22T23:45:09,204 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,46175,1732319105875, state=OPEN 2024-11-22T23:45:09,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:09,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:09,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:45:09,206 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a511e45b3024,46175,1732319105875 2024-11-22T23:45:09,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:09,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:09,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:45:09,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-22T23:45:09,208 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:45:09,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a511e45b3024,46175,1732319105875 in 210 msec 2024-11-22T23:45:09,208 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(1482): Serving as a511e45b3024,44219,1732319109117, RpcServer on a511e45b3024/172.17.0.2:44219, sessionid=0x1014211bd7d0003 2024-11-22T23:45:09,208 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T23:45:09,208 DEBUG [RS:2;a511e45b3024:44219 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a511e45b3024,44219,1732319109117 2024-11-22T23:45:09,209 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,44219,1732319109117' 2024-11-22T23:45:09,209 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T23:45:09,209 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T23:45:09,210 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T23:45:09,210 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T23:45:09,210 DEBUG [RS:2;a511e45b3024:44219 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a511e45b3024,44219,1732319109117 2024-11-22T23:45:09,210 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,44219,1732319109117' 2024-11-22T23:45:09,210 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T23:45:09,210 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T23:45:09,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T23:45:09,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 367 msec 2024-11-22T23:45:09,211 DEBUG [RS:2;a511e45b3024:44219 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T23:45:09,211 INFO [RS:2;a511e45b3024:44219 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T23:45:09,211 INFO [RS:2;a511e45b3024:44219 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T23:45:09,211 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): a511e45b3024,34825,1732319105844 had 2 regions 2024-11-22T23:45:09,212 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a511e45b3024,34825,1732319105844, splitWal=true, meta=true, isMeta: false 2024-11-22T23:45:09,212 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34825,1732319105844-splitting dir is empty, no logs to split. 2024-11-22T23:45:09,212 INFO [PEWorker-3 {}] master.SplitWALManager(105): a511e45b3024,34825,1732319105844 WAL count=0, meta=false 2024-11-22T23:45:09,213 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,34825,1732319105844-splitting dir is empty, no logs to split. 2024-11-22T23:45:09,213 INFO [PEWorker-3 {}] master.SplitWALManager(105): a511e45b3024,34825,1732319105844 WAL count=0, meta=false 2024-11-22T23:45:09,213 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if a511e45b3024,34825,1732319105844 WAL splitting is done? wals=0, meta=false 2024-11-22T23:45:09,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732319106916, region=aa4b9fab2da1a9ae656805c6c4ea954b, ASSIGN}] 2024-11-22T23:45:09,215 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732319106916, region=aa4b9fab2da1a9ae656805c6c4ea954b, ASSIGN 2024-11-22T23:45:09,216 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732319106916, region=aa4b9fab2da1a9ae656805c6c4ea954b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T23:45:09,312 INFO [RS:2;a511e45b3024:44219 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:45:09,313 INFO [RS:2;a511e45b3024:44219 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a511e45b3024%2C44219%2C1732319109117, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,44219,1732319109117, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs, maxLogs=55 2024-11-22T23:45:09,332 INFO [RS:2;a511e45b3024:44219 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/WALs/a511e45b3024,44219,1732319109117/a511e45b3024%2C44219%2C1732319109117.1732319109313 2024-11-22T23:45:09,332 DEBUG [RS:2;a511e45b3024:44219 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:45:09,367 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(204): Hosts are {a511e45b3024=0} racks are {/default-rack=0} 2024-11-22T23:45:09,367 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-22T23:45:09,367 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-22T23:45:09,367 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-22T23:45:09,367 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-22T23:45:09,367 INFO [a511e45b3024:40019 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-22T23:45:09,367 INFO [a511e45b3024:40019 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-22T23:45:09,367 DEBUG [a511e45b3024:40019 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-22T23:45:09,368 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=aa4b9fab2da1a9ae656805c6c4ea954b, regionState=OPENING, regionLocation=a511e45b3024,44219,1732319109117 2024-11-22T23:45:09,369 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a511e45b3024:34825 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a511e45b3024/172.17.0.2:34825 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:45:09,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a511e45b3024,34825,1732319105844, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a511e45b3024,34825,1732319105844, seqNum=-1, error=java.net.ConnectException: Call to address=a511e45b3024:34825 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a511e45b3024/172.17.0.2:34825 2024-11-22T23:45:09,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a511e45b3024,34825,1732319105844, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-22T23:45:09,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a511e45b3024,34825,1732319105844, seqNum=-1 from cache 2024-11-22T23:45:09,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address a511e45b3024:34825 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a511e45b3024/172.17.0.2:34825 2024-11-22T23:45:09,483 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:45:09,483 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,46175,1732319105875, seqNum=-1] 2024-11-22T23:45:09,484 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:45:09,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56745, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:45:09,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732319106916, region=aa4b9fab2da1a9ae656805c6c4ea954b, ASSIGN because future has completed 2024-11-22T23:45:09,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa4b9fab2da1a9ae656805c6c4ea954b, server=a511e45b3024,44219,1732319109117}] 2024-11-22T23:45:09,641 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T23:45:09,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49061, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T23:45:09,644 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:09,645 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => aa4b9fab2da1a9ae656805c6c4ea954b, NAME => 'testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:45:09,645 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732319106916 aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,646 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:45:09,646 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,646 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,646 INFO [StoreOpener-aa4b9fab2da1a9ae656805c6c4ea954b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,647 INFO [StoreOpener-aa4b9fab2da1a9ae656805c6c4ea954b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa4b9fab2da1a9ae656805c6c4ea954b columnFamilyName family 2024-11-22T23:45:09,647 DEBUG [StoreOpener-aa4b9fab2da1a9ae656805c6c4ea954b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:45:09,649 DEBUG [StoreFileOpener-aa4b9fab2da1a9ae656805c6c4ea954b-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b, delay=997 ms 2024-11-22T23:45:09,650 DEBUG [StoreOpener-aa4b9fab2da1a9ae656805c6c4ea954b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b 2024-11-22T23:45:09,650 INFO [StoreOpener-aa4b9fab2da1a9ae656805c6c4ea954b-1 {}] regionserver.HStore(327): Store=aa4b9fab2da1a9ae656805c6c4ea954b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:45:09,650 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,651 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,651 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,652 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,652 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,653 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,653 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened aa4b9fab2da1a9ae656805c6c4ea954b; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74249343, jitterRate=0.10640142858028412}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T23:45:09,653 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,654 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for aa4b9fab2da1a9ae656805c6c4ea954b: Running coprocessor pre-open hook at 1732319109646Writing region info on filesystem at 1732319109646Initializing all the Stores at 1732319109646Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319109646Cleaning up temporary data from old regions at 1732319109652 (+6 ms)Running coprocessor post-open hooks at 1732319109653 (+1 ms)Region opened successfully at 1732319109654 (+1 ms) 2024-11-22T23:45:09,655 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b., pid=11, masterSystemTime=1732319109640 2024-11-22T23:45:09,657 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:09,657 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:09,658 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=aa4b9fab2da1a9ae656805c6c4ea954b, regionState=OPEN, openSeqNum=10, regionLocation=a511e45b3024,44219,1732319109117 2024-11-22T23:45:09,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa4b9fab2da1a9ae656805c6c4ea954b, server=a511e45b3024,44219,1732319109117 because future has completed 2024-11-22T23:45:09,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-22T23:45:09,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure aa4b9fab2da1a9ae656805c6c4ea954b, server=a511e45b3024,44219,1732319109117 in 173 msec 2024-11-22T23:45:09,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-22T23:45:09,664 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server a511e45b3024,34825,1732319105844 after splitting done 2024-11-22T23:45:09,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732319106916, region=aa4b9fab2da1a9ae656805c6c4ea954b, ASSIGN in 448 msec 2024-11-22T23:45:09,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure a511e45b3024,34825,1732319105844, splitWal=true, meta=true in 834 msec 2024-11-22T23:45:09,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T23:45:09,709 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:45:09,709 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:45:09,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:09,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:09,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:45:09,709 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T23:45:09,709 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=794444007, stopped=false 2024-11-22T23:45:09,709 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a511e45b3024,40019,1732319105791 2024-11-22T23:45:09,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:09,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:09,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:45:09,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:09,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:09,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:09,711 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:45:09,712 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:45:09,712 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:45:09,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:09,712 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a511e45b3024,46175,1732319105875' ***** 2024-11-22T23:45:09,712 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T23:45:09,712 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a511e45b3024,44219,1732319109117' ***** 2024-11-22T23:45:09,712 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T23:45:09,713 INFO [RS:1;a511e45b3024:46175 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T23:45:09,713 INFO [RS:2;a511e45b3024:44219 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T23:45:09,713 INFO [RS:1;a511e45b3024:46175 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.persistence 2024-11-22T23:45:09,713 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T23:45:09,713 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T23:45:09,713 INFO [RS:2;a511e45b3024:44219 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e39ccda-0056-c991-21ee-e2098c595347/bucket.persistence 2024-11-22T23:45:09,713 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T23:45:09,713 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T23:45:09,713 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T23:45:09,713 INFO [RS:2;a511e45b3024:44219 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T23:45:09,713 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T23:45:09,713 INFO [RS:1;a511e45b3024:46175 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T23:45:09,714 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:09,714 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:09,714 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T23:45:09,714 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T23:45:09,714 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:45:09,723 DEBUG [RS:2;a511e45b3024:44219 {}] bucket.BucketCache(1487): Thread RS:2;a511e45b3024:44219 started persisting bucket cache to file 2024-11-22T23:45:09,724 DEBUG [RS:2;a511e45b3024:44219 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T23:45:09,724 DEBUG [RS:1;a511e45b3024:46175 {}] bucket.BucketCache(1487): Thread RS:1;a511e45b3024:46175 started persisting bucket cache to file 2024-11-22T23:45:09,724 DEBUG [RS:2;a511e45b3024:44219 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T23:45:09,724 DEBUG [RS:1;a511e45b3024:46175 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T23:45:09,724 DEBUG [RS:1;a511e45b3024:46175 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-22T23:45:09,727 DEBUG [RS:1;a511e45b3024:46175 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-22T23:45:09,727 DEBUG [RS:1;a511e45b3024:46175 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732319109724 2024-11-22T23:45:09,727 DEBUG [RS:1;a511e45b3024:46175 {}] bucket.BucketCache(1509): Thread RS:1;a511e45b3024:46175 finished persisting bucket cache to file, renaming 2024-11-22T23:45:09,727 DEBUG [RS:2;a511e45b3024:44219 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-22T23:45:09,727 DEBUG [RS:2;a511e45b3024:44219 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732319109723 2024-11-22T23:45:09,727 DEBUG [RS:2;a511e45b3024:44219 {}] bucket.BucketCache(1509): Thread RS:2;a511e45b3024:44219 finished persisting bucket cache to file, renaming 2024-11-22T23:45:09,727 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T23:45:09,728 INFO [RS:1;a511e45b3024:46175 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T23:45:09,728 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T23:45:09,728 INFO [RS:1;a511e45b3024:46175 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T23:45:09,728 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(959): stopping server a511e45b3024,46175,1732319105875 2024-11-22T23:45:09,728 INFO [RS:1;a511e45b3024:46175 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:45:09,728 INFO [RS:1;a511e45b3024:46175 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a511e45b3024:46175. 2024-11-22T23:45:09,728 DEBUG [RS:1;a511e45b3024:46175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:45:09,728 DEBUG [RS:1;a511e45b3024:46175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:09,728 INFO [RS:1;a511e45b3024:46175 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T23:45:09,728 INFO [RS:1;a511e45b3024:46175 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T23:45:09,728 INFO [RS:1;a511e45b3024:46175 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T23:45:09,728 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T23:45:09,728 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T23:45:09,728 DEBUG [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T23:45:09,728 DEBUG [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:45:09,729 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T23:45:09,729 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:45:09,729 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:45:09,729 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:45:09,729 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:45:09,729 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:45:09,729 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-22T23:45:09,729 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T23:45:09,729 INFO [RS:2;a511e45b3024:44219 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T23:45:09,729 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T23:45:09,729 INFO [RS:2;a511e45b3024:44219 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T23:45:09,729 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(3091): Received CLOSE for aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,730 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(959): stopping server a511e45b3024,44219,1732319109117 2024-11-22T23:45:09,730 INFO [RS:2;a511e45b3024:44219 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:45:09,730 INFO [RS:2;a511e45b3024:44219 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a511e45b3024:44219. 2024-11-22T23:45:09,730 DEBUG [RS:2;a511e45b3024:44219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:45:09,730 DEBUG [RS:2;a511e45b3024:44219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:09,730 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T23:45:09,730 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T23:45:09,730 DEBUG [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(1325): Online Regions={aa4b9fab2da1a9ae656805c6c4ea954b=testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b.} 2024-11-22T23:45:09,730 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aa4b9fab2da1a9ae656805c6c4ea954b, disabling compactions & flushes 2024-11-22T23:45:09,730 DEBUG [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(1351): Waiting on aa4b9fab2da1a9ae656805c6c4ea954b 2024-11-22T23:45:09,730 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:09,730 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:09,730 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. after waiting 0 ms 2024-11-22T23:45:09,730 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:09,731 DEBUG [StoreFileCloser-aa4b9fab2da1a9ae656805c6c4ea954b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/family/ed2631e40f6e4b8ebc527582241ec61b 2024-11-22T23:45:09,736 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/default/testBlockEvictionOnGracefulStop1732319106916/aa4b9fab2da1a9ae656805c6c4ea954b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-22T23:45:09,736 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:09,737 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aa4b9fab2da1a9ae656805c6c4ea954b: Waiting for close lock at 1732319109730Running coprocessor pre-close hooks at 1732319109730Disabling compacts and flushes for region at 1732319109730Disabling writes for close at 1732319109730Writing region close event to WAL at 1732319109731 (+1 ms)Running coprocessor post-close hooks at 1732319109736 (+5 ms)Closed at 1732319109736 2024-11-22T23:45:09,737 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b. 2024-11-22T23:45:09,741 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 40e9464ad0164582a34e2e751d6a0329 2024-11-22T23:45:09,745 INFO [regionserver/a511e45b3024:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:45:09,754 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/info/40e9464ad0164582a34e2e751d6a0329 is 209, key is testBlockEvictionOnGracefulStop1732319106916,,1732319106921.aa4b9fab2da1a9ae656805c6c4ea954b./info:regioninfo/1732319109658/Put/seqid=0 2024-11-22T23:45:09,755 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/info/40e9464ad0164582a34e2e751d6a0329 2024-11-22T23:45:09,757 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/.tmp/info/40e9464ad0164582a34e2e751d6a0329 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/40e9464ad0164582a34e2e751d6a0329 2024-11-22T23:45:09,758 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/40e9464ad0164582a34e2e751d6a0329, delay=981 ms 2024-11-22T23:45:09,759 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/40e9464ad0164582a34e2e751d6a0329, entries=8, sequenceid=19, filesize=6.6 K 2024-11-22T23:45:09,760 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false 2024-11-22T23:45:09,761 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/b8e03362d4ea4663867d220e66115796 2024-11-22T23:45:09,761 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/info/40e9464ad0164582a34e2e751d6a0329 2024-11-22T23:45:09,761 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/ns/bda423a0c6ec48239ce7daad7214f599 2024-11-22T23:45:09,762 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/table/cf8923cd3a6a4395ad911b7927d9951d 2024-11-22T23:45:09,766 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-22T23:45:09,766 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T23:45:09,766 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:45:09,766 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732319109728Running coprocessor pre-close hooks at 1732319109729 (+1 ms)Disabling compacts and flushes for region at 1732319109729Disabling writes for close at 1732319109729Obtaining lock to block concurrent updates at 1732319109729Preparing flush snapshotting stores in 1588230740 at 1732319109729Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732319109730 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732319109730Flushing 1588230740/info: creating writer at 1732319109730Flushing 1588230740/info: appending metadata at 1732319109754 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732319109754Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42226024: reopening flushed file at 1732319109757 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false at 1732319109760 (+3 ms)Writing region close event to WAL at 1732319109762 (+2 ms)Running coprocessor post-close hooks at 1732319109766 (+4 ms)Closed at 1732319109766 2024-11-22T23:45:09,767 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T23:45:09,778 INFO [regionserver/a511e45b3024:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:45:09,929 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(976): stopping server a511e45b3024,46175,1732319105875; all regions closed. 2024-11-22T23:45:09,930 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(976): stopping server a511e45b3024,44219,1732319109117; all regions closed. 2024-11-22T23:45:09,931 DEBUG [RS:1;a511e45b3024:46175 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs 2024-11-22T23:45:09,931 INFO [RS:1;a511e45b3024:46175 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C46175%2C1732319105875.meta:.meta(num 1732319109155) 2024-11-22T23:45:09,932 DEBUG [RS:2;a511e45b3024:44219 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs 2024-11-22T23:45:09,932 INFO [RS:2;a511e45b3024:44219 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C44219%2C1732319109117:(num 1732319109313) 2024-11-22T23:45:09,932 DEBUG [RS:2;a511e45b3024:44219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:09,932 INFO [RS:2;a511e45b3024:44219 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:45:09,932 INFO [RS:2;a511e45b3024:44219 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:45:09,932 INFO [RS:2;a511e45b3024:44219 {}] hbase.ChoreService(370): Chore service for: regionserver/a511e45b3024:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T23:45:09,933 INFO [RS:2;a511e45b3024:44219 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T23:45:09,933 INFO [regionserver/a511e45b3024:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:45:09,933 INFO [RS:2;a511e45b3024:44219 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T23:45:09,933 INFO [RS:2;a511e45b3024:44219 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T23:45:09,933 DEBUG [RS:1;a511e45b3024:46175 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/oldWALs 2024-11-22T23:45:09,933 INFO [RS:2;a511e45b3024:44219 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:45:09,933 INFO [RS:1;a511e45b3024:46175 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C46175%2C1732319105875:(num 1732319106381) 2024-11-22T23:45:09,933 DEBUG [RS:1;a511e45b3024:46175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:45:09,933 INFO [RS:1;a511e45b3024:46175 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:45:09,933 INFO [RS:2;a511e45b3024:44219 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44219 2024-11-22T23:45:09,933 INFO [RS:1;a511e45b3024:46175 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:45:09,933 INFO [RS:1;a511e45b3024:46175 {}] hbase.ChoreService(370): Chore service for: regionserver/a511e45b3024:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T23:45:09,934 INFO [RS:1;a511e45b3024:46175 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:45:09,934 INFO [regionserver/a511e45b3024:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:45:09,934 INFO [RS:1;a511e45b3024:46175 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46175 2024-11-22T23:45:09,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a511e45b3024,44219,1732319109117 2024-11-22T23:45:09,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:45:09,936 INFO [RS:2;a511e45b3024:44219 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:45:09,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a511e45b3024,46175,1732319105875 2024-11-22T23:45:09,936 INFO [RS:1;a511e45b3024:46175 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:45:09,938 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a511e45b3024,46175,1732319105875] 2024-11-22T23:45:09,940 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a511e45b3024,46175,1732319105875 already deleted, retry=false 2024-11-22T23:45:09,940 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a511e45b3024,46175,1732319105875 expired; onlineServers=1 2024-11-22T23:45:09,940 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a511e45b3024,44219,1732319109117] 2024-11-22T23:45:09,942 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a511e45b3024,44219,1732319109117 already deleted, retry=false 2024-11-22T23:45:09,942 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a511e45b3024,44219,1732319109117 expired; onlineServers=0 2024-11-22T23:45:09,942 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a511e45b3024,40019,1732319105791' ***** 2024-11-22T23:45:09,942 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T23:45:09,942 INFO [M:0;a511e45b3024:40019 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:45:09,942 INFO [M:0;a511e45b3024:40019 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:45:09,942 DEBUG [M:0;a511e45b3024:40019 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T23:45:09,942 DEBUG [M:0;a511e45b3024:40019 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T23:45:09,942 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T23:45:09,942 DEBUG [master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732319106124 {}] cleaner.HFileCleaner(306): Exit Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732319106124,5,FailOnTimeoutGroup] 2024-11-22T23:45:09,942 DEBUG [master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732319106124 {}] cleaner.HFileCleaner(306): Exit Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732319106124,5,FailOnTimeoutGroup] 2024-11-22T23:45:09,943 INFO [M:0;a511e45b3024:40019 {}] hbase.ChoreService(370): Chore service for: master/a511e45b3024:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T23:45:09,943 INFO [M:0;a511e45b3024:40019 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:45:09,943 DEBUG [M:0;a511e45b3024:40019 {}] master.HMaster(1795): Stopping service threads 2024-11-22T23:45:09,943 INFO [M:0;a511e45b3024:40019 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T23:45:09,943 INFO [M:0;a511e45b3024:40019 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:45:09,943 INFO [M:0;a511e45b3024:40019 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:45:09,944 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T23:45:09,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T23:45:09,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:45:09,945 DEBUG [M:0;a511e45b3024:40019 {}] zookeeper.ZKUtil(347): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T23:45:09,945 WARN [M:0;a511e45b3024:40019 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T23:45:09,946 INFO [M:0;a511e45b3024:40019 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/.lastflushedseqids 2024-11-22T23:45:09,949 INFO [M:0;a511e45b3024:40019 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:45:09,950 INFO [M:0;a511e45b3024:40019 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T23:45:09,950 DEBUG [M:0;a511e45b3024:40019 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:45:09,950 INFO [M:0;a511e45b3024:40019 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:09,950 DEBUG [M:0;a511e45b3024:40019 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:09,950 DEBUG [M:0;a511e45b3024:40019 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:45:09,950 DEBUG [M:0;a511e45b3024:40019 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:09,950 INFO [M:0;a511e45b3024:40019 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-22T23:45:09,961 DEBUG [M:0;a511e45b3024:40019 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f4c5f718ddd44a82a47df6b5c7a1348a 2024-11-22T23:45:09,979 DEBUG [M:0;a511e45b3024:40019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4c5f718ddd44a82a47df6b5c7a1348a is 82, key is hbase:meta,,1/info:regioninfo/1732319109203/Put/seqid=0 2024-11-22T23:45:09,980 INFO [M:0;a511e45b3024:40019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4c5f718ddd44a82a47df6b5c7a1348a 2024-11-22T23:45:09,988 DEBUG [M:0;a511e45b3024:40019 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5ca22533656746aabe64c2c4c576ce77 2024-11-22T23:45:10,002 DEBUG [M:0;a511e45b3024:40019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5ca22533656746aabe64c2c4c576ce77 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732319107300/Put/seqid=0 2024-11-22T23:45:10,003 INFO [M:0;a511e45b3024:40019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5ca22533656746aabe64c2c4c576ce77 2024-11-22T23:45:10,010 DEBUG [M:0;a511e45b3024:40019 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7d4a718f645b4803903fe2724a9a3ad0 2024-11-22T23:45:10,025 DEBUG [M:0;a511e45b3024:40019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7d4a718f645b4803903fe2724a9a3ad0 is 69, key is a511e45b3024,44219,1732319109117/rs:state/1732319109160/Put/seqid=0 2024-11-22T23:45:10,026 INFO [M:0;a511e45b3024:40019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7d4a718f645b4803903fe2724a9a3ad0 2024-11-22T23:45:10,028 INFO [M:0;a511e45b3024:40019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7d4a718f645b4803903fe2724a9a3ad0 2024-11-22T23:45:10,029 DEBUG [M:0;a511e45b3024:40019 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4c5f718ddd44a82a47df6b5c7a1348a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f4c5f718ddd44a82a47df6b5c7a1348a 2024-11-22T23:45:10,031 INFO [M:0;a511e45b3024:40019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f4c5f718ddd44a82a47df6b5c7a1348a, entries=14, sequenceid=102, filesize=5.9 K 2024-11-22T23:45:10,031 DEBUG [M:0;a511e45b3024:40019 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5ca22533656746aabe64c2c4c576ce77 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5ca22533656746aabe64c2c4c576ce77 2024-11-22T23:45:10,033 INFO [M:0;a511e45b3024:40019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5ca22533656746aabe64c2c4c576ce77, entries=11, sequenceid=102, filesize=6.5 K 2024-11-22T23:45:10,033 DEBUG [M:0;a511e45b3024:40019 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7d4a718f645b4803903fe2724a9a3ad0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7d4a718f645b4803903fe2724a9a3ad0 2024-11-22T23:45:10,035 INFO [M:0;a511e45b3024:40019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7d4a718f645b4803903fe2724a9a3ad0 2024-11-22T23:45:10,035 INFO [M:0;a511e45b3024:40019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ac2069c-c7b9-8688-5534-a3890607ed28/b15f5763-0e21-2551-80bc-817209050532/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7d4a718f645b4803903fe2724a9a3ad0, entries=3, sequenceid=102, filesize=5.3 K 2024-11-22T23:45:10,035 INFO [M:0;a511e45b3024:40019 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=102, compaction requested=false 2024-11-22T23:45:10,037 INFO [M:0;a511e45b3024:40019 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:45:10,037 DEBUG [M:0;a511e45b3024:40019 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732319109950Disabling compacts and flushes for region at 1732319109950Disabling writes for close at 1732319109950Obtaining lock to block concurrent updates at 1732319109950Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732319109950Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732319109951 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732319109951Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732319109951Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732319109978 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732319109978Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732319109982 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732319110002 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732319110002Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732319110004 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732319110024 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732319110024Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b33b83: reopening flushed file at 1732319110029 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5af9f35a: reopening flushed file at 1732319110031 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2604ea37: reopening flushed file at 1732319110033 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=102, compaction requested=false at 1732319110035 (+2 ms)Writing region close event to WAL at 1732319110037 (+2 ms)Closed at 1732319110037 2024-11-22T23:45:10,038 INFO [M:0;a511e45b3024:40019 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T23:45:10,038 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:45:10,038 INFO [M:0;a511e45b3024:40019 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40019 2024-11-22T23:45:10,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:10,039 INFO [RS:2;a511e45b3024:44219 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:45:10,039 INFO [RS:2;a511e45b3024:44219 {}] regionserver.HRegionServer(1031): Exiting; stopping=a511e45b3024,44219,1732319109117; zookeeper connection closed. 2024-11-22T23:45:10,039 INFO [M:0;a511e45b3024:40019 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:45:10,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44219-0x1014211bd7d0003, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:10,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:10,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46175-0x1014211bd7d0002, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:10,039 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@691c0667 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@691c0667 2024-11-22T23:45:10,040 INFO [RS:1;a511e45b3024:46175 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:45:10,040 INFO [RS:1;a511e45b3024:46175 {}] regionserver.HRegionServer(1031): Exiting; stopping=a511e45b3024,46175,1732319105875; zookeeper connection closed. 2024-11-22T23:45:10,040 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@33e17912 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@33e17912 2024-11-22T23:45:10,041 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-22T23:45:10,141 INFO [M:0;a511e45b3024:40019 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:45:10,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:10,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x1014211bd7d0000, quorum=127.0.0.1:55949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:45:10,144 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T23:45:10,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T23:45:10,146 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:45:10,153 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=236 (was 248), ProcessCount=11 (was 11), AvailableMemoryMB=9212 (was 9293)