2024-11-20 14:55:06,018 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 14:55:06,051 main DEBUG Took 0.029831 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 14:55:06,051 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 14:55:06,052 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 14:55:06,053 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 14:55:06,055 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,066 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 14:55:06,085 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,089 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,091 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,092 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,094 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,095 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,097 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,098 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,098 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,099 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,100 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,101 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,102 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,102 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 14:55:06,103 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,103 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 14:55:06,105 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 14:55:06,107 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 14:55:06,110 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 14:55:06,111 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 14:55:06,112 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 14:55:06,113 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 14:55:06,125 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 14:55:06,129 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 14:55:06,131 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 14:55:06,132 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 14:55:06,132 main DEBUG createAppenders(={Console}) 2024-11-20 14:55:06,133 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-20 14:55:06,134 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 14:55:06,134 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-20 14:55:06,135 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 14:55:06,135 main DEBUG OutputStream closed 2024-11-20 14:55:06,136 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 14:55:06,136 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 14:55:06,136 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-20 14:55:06,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 14:55:06,222 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 14:55:06,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 14:55:06,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 14:55:06,224 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 14:55:06,224 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 14:55:06,225 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 14:55:06,225 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 14:55:06,225 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 14:55:06,225 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 14:55:06,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 14:55:06,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 14:55:06,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 14:55:06,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 14:55:06,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 14:55:06,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 14:55:06,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 14:55:06,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 14:55:06,230 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 14:55:06,230 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-20 14:55:06,231 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 14:55:06,231 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-20T14:55:06,243 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-20 14:55:06,245 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 14:55:06,246 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T14:55:06,480 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11 2024-11-20T14:55:06,508 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=11344 2024-11-20T14:55:06,607 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T14:55:06,700 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:06,716 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/cluster_312eec8b-e71d-f93f-e2e8-9ccb2ad58dcd, deleteOnExit=true 2024-11-20T14:55:06,830 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/cluster_312eec8b-e71d-f93f-e2e8-9ccb2ad58dcd/zookeeper_0, clientPort=63621, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/cluster_312eec8b-e71d-f93f-e2e8-9ccb2ad58dcd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/cluster_312eec8b-e71d-f93f-e2e8-9ccb2ad58dcd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T14:55:06,850 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63621 2024-11-20T14:55:06,851 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:06,961 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 with version=8 2024-11-20T14:55:06,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/hbase-staging 2024-11-20T14:55:07,058 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T14:55:07,319 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T14:55:07,331 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:07,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:07,340 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T14:55:07,340 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:07,340 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T14:55:07,480 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T14:55:07,562 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T14:55:07,606 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T14:55:07,617 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T14:55:07,621 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T14:55:07,659 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 96004 (auto-detected) 2024-11-20T14:55:07,660 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T14:55:07,690 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45021 2024-11-20T14:55:07,718 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45021 connecting to ZooKeeper ensemble=127.0.0.1:63621 2024-11-20T14:55:07,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:450210x0, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T14:55:07,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45021-0x1015a0776f40000 connected 2024-11-20T14:55:07,900 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:07,902 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:07,910 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:07,913 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8, hbase.cluster.distributed=false 2024-11-20T14:55:07,936 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T14:55:07,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45021 2024-11-20T14:55:07,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45021 2024-11-20T14:55:07,942 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45021 2024-11-20T14:55:07,942 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45021 2024-11-20T14:55:07,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45021 2024-11-20T14:55:08,014 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T14:55:08,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:08,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:08,016 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T14:55:08,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:08,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T14:55:08,019 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T14:55:08,022 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T14:55:08,023 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44997 2024-11-20T14:55:08,024 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44997 connecting to ZooKeeper ensemble=127.0.0.1:63621 2024-11-20T14:55:08,025 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:08,026 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:08,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:449970x0, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T14:55:08,059 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44997-0x1015a0776f40001 connected 2024-11-20T14:55:08,060 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:08,067 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T14:55:08,080 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/bucket.cache 2024-11-20T14:55:08,087 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T14:55:08,088 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T14:55:08,091 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T14:55:08,093 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T14:55:08,094 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T14:55:08,100 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T14:55:08,105 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T14:55:08,106 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/bucket.persistence is complete. 2024-11-20T14:55:08,107 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T14:55:08,110 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T14:55:08,117 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T14:55:08,118 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44997 2024-11-20T14:55:08,119 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44997 2024-11-20T14:55:08,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44997 2024-11-20T14:55:08,130 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44997 2024-11-20T14:55:08,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44997 2024-11-20T14:55:08,157 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T14:55:08,157 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:08,157 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:08,158 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T14:55:08,158 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:08,158 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T14:55:08,159 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T14:55:08,159 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T14:55:08,160 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44165 2024-11-20T14:55:08,163 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44165 connecting to ZooKeeper ensemble=127.0.0.1:63621 2024-11-20T14:55:08,164 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:08,165 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:08,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:441650x0, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T14:55:08,205 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44165-0x1015a0776f40002 connected 2024-11-20T14:55:08,206 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:08,208 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T14:55:08,209 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/bucket.cache 2024-11-20T14:55:08,209 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T14:55:08,210 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T14:55:08,210 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T14:55:08,211 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T14:55:08,211 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T14:55:08,211 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T14:55:08,212 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T14:55:08,212 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/bucket.persistence is complete. 2024-11-20T14:55:08,212 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T14:55:08,214 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T14:55:08,217 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T14:55:08,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44165 2024-11-20T14:55:08,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44165 2024-11-20T14:55:08,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44165 2024-11-20T14:55:08,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44165 2024-11-20T14:55:08,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44165 2024-11-20T14:55:08,254 DEBUG [M:0;1a15ecfd95f4:45021 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a15ecfd95f4:45021 2024-11-20T14:55:08,255 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1a15ecfd95f4,45021,1732114507133 2024-11-20T14:55:08,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:08,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:08,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:08,290 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a15ecfd95f4,45021,1732114507133 2024-11-20T14:55:08,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T14:55:08,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T14:55:08,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:08,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:08,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:08,335 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T14:55:08,336 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a15ecfd95f4,45021,1732114507133 from backup master directory 2024-11-20T14:55:08,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:08,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a15ecfd95f4,45021,1732114507133 2024-11-20T14:55:08,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:08,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:08,346 WARN [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T14:55:08,346 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a15ecfd95f4,45021,1732114507133 2024-11-20T14:55:08,349 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T14:55:08,351 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T14:55:08,361 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/hbase.id] with ID: 9158d560-6048-49a1-9ba6-ee08dcbba61e 2024-11-20T14:55:08,362 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/.tmp/hbase.id 2024-11-20T14:55:08,367 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/hbase.id] 2024-11-20T14:55:08,402 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:08,410 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T14:55:08,432 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-20T14:55:08,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:08,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:08,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:08,491 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T14:55:08,495 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T14:55:08,542 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T14:55:08,549 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T14:55:08,599 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store 2024-11-20T14:55:08,617 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T14:55:08,620 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:08,622 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T14:55:08,622 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:08,622 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:08,623 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T14:55:08,624 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:08,624 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:08,624 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732114508621Disabling compacts and flushes for region at 1732114508621Disabling writes for close at 1732114508624 (+3 ms)Writing region close event to WAL at 1732114508624Closed at 1732114508624 2024-11-20T14:55:08,625 WARN [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/.initializing 2024-11-20T14:55:08,625 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/WALs/1a15ecfd95f4,45021,1732114507133 2024-11-20T14:55:08,636 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T14:55:08,647 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a15ecfd95f4%2C45021%2C1732114507133, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/WALs/1a15ecfd95f4,45021,1732114507133, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/oldWALs, maxLogs=10 2024-11-20T14:55:08,681 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/WALs/1a15ecfd95f4,45021,1732114507133/1a15ecfd95f4%2C45021%2C1732114507133.1732114508651 2024-11-20T14:55:08,682 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T14:55:08,683 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T14:55:08,683 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:08,686 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,687 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,724 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,772 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T14:55:08,778 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:08,781 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:08,782 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,789 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T14:55:08,789 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:08,790 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:08,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,798 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T14:55:08,799 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:08,799 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:08,800 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T14:55:08,806 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:08,807 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:08,807 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,812 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,812 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,816 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,817 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,820 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T14:55:08,823 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:08,830 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T14:55:08,832 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70373053, jitterRate=0.04864020645618439}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T14:55:08,843 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732114508698Initializing all the Stores at 1732114508699 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114508699Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114508700 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114508700Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114508700Cleaning up temporary data from old regions at 1732114508817 (+117 ms)Region opened successfully at 1732114508843 (+26 ms) 2024-11-20T14:55:08,846 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T14:55:08,881 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1af41e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T14:55:08,916 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T14:55:08,934 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T14:55:08,935 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T14:55:08,939 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T14:55:08,940 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T14:55:08,948 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T14:55:08,948 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T14:55:08,980 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T14:55:08,991 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T14:55:09,029 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T14:55:09,032 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T14:55:09,034 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T14:55:09,045 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T14:55:09,048 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T14:55:09,052 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T14:55:09,062 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T14:55:09,064 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T14:55:09,074 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T14:55:09,091 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T14:55:09,100 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T14:55:09,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:09,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:09,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:09,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:09,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:09,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:09,124 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1a15ecfd95f4,45021,1732114507133, sessionid=0x1015a0776f40000, setting cluster-up flag (Was=false) 2024-11-20T14:55:09,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:09,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:09,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:09,179 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T14:55:09,181 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a15ecfd95f4,45021,1732114507133 2024-11-20T14:55:09,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:09,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:09,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:09,262 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T14:55:09,264 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a15ecfd95f4,45021,1732114507133 2024-11-20T14:55:09,269 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T14:55:09,338 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(746): ClusterId : 9158d560-6048-49a1-9ba6-ee08dcbba61e 2024-11-20T14:55:09,338 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(746): ClusterId : 9158d560-6048-49a1-9ba6-ee08dcbba61e 2024-11-20T14:55:09,340 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T14:55:09,340 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T14:55:09,345 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T14:55:09,356 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T14:55:09,356 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T14:55:09,356 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T14:55:09,356 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T14:55:09,360 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T14:55:09,368 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T14:55:09,372 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T14:55:09,372 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T14:55:09,373 DEBUG [RS:1;1a15ecfd95f4:44165 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@410fb47a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T14:55:09,373 DEBUG [RS:0;1a15ecfd95f4:44997 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4409fb1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T14:55:09,374 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a15ecfd95f4,45021,1732114507133 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T14:55:09,386 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T14:55:09,387 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T14:55:09,387 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T14:55:09,387 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T14:55:09,388 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a15ecfd95f4:0, corePoolSize=10, maxPoolSize=10 2024-11-20T14:55:09,388 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,388 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T14:55:09,388 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,390 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732114539390 2024-11-20T14:55:09,390 DEBUG [RS:0;1a15ecfd95f4:44997 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a15ecfd95f4:44997 2024-11-20T14:55:09,390 DEBUG [RS:1;1a15ecfd95f4:44165 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1a15ecfd95f4:44165 2024-11-20T14:55:09,391 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T14:55:09,392 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T14:55:09,393 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T14:55:09,393 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T14:55:09,393 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T14:55:09,393 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T14:55:09,393 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T14:55:09,394 DEBUG [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T14:55:09,394 DEBUG [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T14:55:09,394 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T14:55:09,395 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T14:55:09,396 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T14:55:09,396 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T14:55:09,396 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T14:55:09,399 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a15ecfd95f4,45021,1732114507133 with port=44165, startcode=1732114508156 2024-11-20T14:55:09,399 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a15ecfd95f4,45021,1732114507133 with port=44997, startcode=1732114507984 2024-11-20T14:55:09,399 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:09,397 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,400 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T14:55:09,407 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T14:55:09,409 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T14:55:09,410 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T14:55:09,412 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T14:55:09,413 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T14:55:09,414 DEBUG [RS:1;1a15ecfd95f4:44165 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T14:55:09,414 DEBUG [RS:0;1a15ecfd95f4:44997 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T14:55:09,417 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.large.0-1732114509414,5,FailOnTimeoutGroup] 2024-11-20T14:55:09,418 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.small.0-1732114509417,5,FailOnTimeoutGroup] 2024-11-20T14:55:09,419 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,419 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T14:55:09,420 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,420 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,436 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T14:55:09,436 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 2024-11-20T14:55:09,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:09,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T14:55:09,459 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T14:55:09,459 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:09,459 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:09,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T14:55:09,467 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T14:55:09,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:09,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:09,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T14:55:09,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T14:55:09,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:09,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:09,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T14:55:09,492 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T14:55:09,492 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57987, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T14:55:09,492 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35151, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T14:55:09,493 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:09,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:09,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T14:55:09,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740 2024-11-20T14:55:09,495 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740 2024-11-20T14:55:09,496 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T14:55:09,496 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T14:55:09,496 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T14:55:09,497 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T14:55:09,500 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45021 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:09,503 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45021 {}] master.ServerManager(517): Registering regionserver=1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:09,515 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45021 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:09,515 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45021 {}] master.ServerManager(517): Registering regionserver=1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:09,520 DEBUG [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 2024-11-20T14:55:09,520 DEBUG [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 2024-11-20T14:55:09,520 DEBUG [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T14:55:09,520 DEBUG [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T14:55:09,520 DEBUG [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T14:55:09,520 DEBUG [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T14:55:09,520 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T14:55:09,521 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72289388, jitterRate=0.07719582319259644}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T14:55:09,525 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732114509447Initializing all the Stores at 1732114509447Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114509448 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114509448Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114509448Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114509448Cleaning up temporary data from old regions at 1732114509496 (+48 ms)Region opened successfully at 1732114509525 (+29 ms) 2024-11-20T14:55:09,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T14:55:09,526 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T14:55:09,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T14:55:09,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T14:55:09,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T14:55:09,527 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T14:55:09,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732114509526Disabling compacts and flushes for region at 1732114509526Disabling writes for close at 1732114509526Writing region close event to WAL at 1732114509527 (+1 ms)Closed at 1732114509527 2024-11-20T14:55:09,529 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T14:55:09,530 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T14:55:09,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T14:55:09,549 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T14:55:09,551 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T14:55:09,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T14:55:09,585 DEBUG [RS:0;1a15ecfd95f4:44997 {}] zookeeper.ZKUtil(111): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:09,585 WARN [RS:0;1a15ecfd95f4:44997 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T14:55:09,585 INFO [RS:0;1a15ecfd95f4:44997 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T14:55:09,586 DEBUG [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:09,587 DEBUG [RS:1;1a15ecfd95f4:44165 {}] zookeeper.ZKUtil(111): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:09,587 WARN [RS:1;1a15ecfd95f4:44165 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T14:55:09,587 INFO [RS:1;1a15ecfd95f4:44165 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T14:55:09,588 DEBUG [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:09,593 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a15ecfd95f4,44997,1732114507984] 2024-11-20T14:55:09,594 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a15ecfd95f4,44165,1732114508156] 2024-11-20T14:55:09,630 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T14:55:09,637 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T14:55:09,662 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T14:55:09,667 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T14:55:09,682 INFO [RS:1;1a15ecfd95f4:44165 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T14:55:09,682 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,688 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T14:55:09,690 INFO [RS:0;1a15ecfd95f4:44997 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T14:55:09,691 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,695 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T14:55:09,697 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,697 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T14:55:09,698 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,698 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,698 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,698 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T14:55:09,698 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,699 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,698 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,699 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T14:55:09,699 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,699 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,699 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,699 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,699 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,699 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,699 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,699 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,700 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,700 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,700 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T14:55:09,700 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,700 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,700 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T14:55:09,700 DEBUG [RS:1;1a15ecfd95f4:44165 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T14:55:09,700 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,700 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,701 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,701 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,701 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:09,701 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T14:55:09,701 DEBUG [RS:0;1a15ecfd95f4:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T14:55:09,703 WARN [1a15ecfd95f4:45021 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T14:55:09,720 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,720 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,721 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,721 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,721 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,721 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,44165,1732114508156-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T14:55:09,739 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,740 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,740 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,743 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,743 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,743 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,44997,1732114507984-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T14:55:09,756 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T14:55:09,758 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,44165,1732114508156-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,759 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,759 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.Replication(171): 1a15ecfd95f4,44165,1732114508156 started 2024-11-20T14:55:09,767 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T14:55:09,767 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,44997,1732114507984-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,767 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,767 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.Replication(171): 1a15ecfd95f4,44997,1732114507984 started 2024-11-20T14:55:09,785 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,786 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(1482): Serving as 1a15ecfd95f4,44165,1732114508156, RpcServer on 1a15ecfd95f4/172.17.0.2:44165, sessionid=0x1015a0776f40002 2024-11-20T14:55:09,788 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T14:55:09,788 DEBUG [RS:1;1a15ecfd95f4:44165 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:09,788 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,44165,1732114508156' 2024-11-20T14:55:09,789 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T14:55:09,790 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T14:55:09,791 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:09,791 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T14:55:09,791 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T14:55:09,792 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(1482): Serving as 1a15ecfd95f4,44997,1732114507984, RpcServer on 1a15ecfd95f4/172.17.0.2:44997, sessionid=0x1015a0776f40001 2024-11-20T14:55:09,792 DEBUG [RS:1;1a15ecfd95f4:44165 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:09,792 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,44165,1732114508156' 2024-11-20T14:55:09,792 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T14:55:09,792 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T14:55:09,792 DEBUG [RS:0;1a15ecfd95f4:44997 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:09,792 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,44997,1732114507984' 2024-11-20T14:55:09,792 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T14:55:09,793 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T14:55:09,793 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T14:55:09,793 DEBUG [RS:1;1a15ecfd95f4:44165 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T14:55:09,794 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T14:55:09,794 INFO [RS:1;1a15ecfd95f4:44165 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T14:55:09,794 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T14:55:09,794 INFO [RS:1;1a15ecfd95f4:44165 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T14:55:09,794 DEBUG [RS:0;1a15ecfd95f4:44997 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:09,794 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,44997,1732114507984' 2024-11-20T14:55:09,794 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T14:55:09,795 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T14:55:09,796 DEBUG [RS:0;1a15ecfd95f4:44997 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T14:55:09,796 INFO [RS:0;1a15ecfd95f4:44997 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T14:55:09,796 INFO [RS:0;1a15ecfd95f4:44997 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T14:55:09,899 INFO [RS:1;1a15ecfd95f4:44165 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T14:55:09,899 INFO [RS:0;1a15ecfd95f4:44997 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T14:55:09,900 INFO [RS:1;1a15ecfd95f4:44165 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1a15ecfd95f4%2C44165%2C1732114508156, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,44165,1732114508156, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs, maxLogs=55 2024-11-20T14:55:09,900 INFO [RS:0;1a15ecfd95f4:44997 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1a15ecfd95f4%2C44997%2C1732114507984, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,44997,1732114507984, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs, maxLogs=55 2024-11-20T14:55:09,926 INFO [RS:0;1a15ecfd95f4:44997 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,44997,1732114507984/1a15ecfd95f4%2C44997%2C1732114507984.1732114509904 2024-11-20T14:55:09,926 INFO [RS:1;1a15ecfd95f4:44165 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,44165,1732114508156/1a15ecfd95f4%2C44165%2C1732114508156.1732114509904 2024-11-20T14:55:09,926 DEBUG [RS:0;1a15ecfd95f4:44997 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T14:55:09,935 DEBUG [RS:1;1a15ecfd95f4:44165 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T14:55:09,957 DEBUG [1a15ecfd95f4:45021 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T14:55:09,969 DEBUG [1a15ecfd95f4:45021 {}] balancer.BalancerClusterState(204): Hosts are {1a15ecfd95f4=0} racks are {/default-rack=0} 2024-11-20T14:55:09,972 DEBUG [1a15ecfd95f4:45021 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T14:55:09,972 DEBUG [1a15ecfd95f4:45021 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T14:55:09,972 DEBUG [1a15ecfd95f4:45021 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T14:55:09,972 DEBUG [1a15ecfd95f4:45021 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T14:55:09,972 INFO [1a15ecfd95f4:45021 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T14:55:09,972 INFO [1a15ecfd95f4:45021 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T14:55:09,972 DEBUG [1a15ecfd95f4:45021 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T14:55:09,980 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:09,985 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,44997,1732114507984, state=OPENING 2024-11-20T14:55:10,008 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T14:55:10,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:10,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:10,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:10,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:10,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:10,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:10,020 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T14:55:10,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,44997,1732114507984}] 2024-11-20T14:55:10,197 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T14:55:10,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50539, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T14:55:10,209 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T14:55:10,209 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T14:55:10,210 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T14:55:10,212 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1a15ecfd95f4%2C44997%2C1732114507984.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,44997,1732114507984, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs, maxLogs=55 2024-11-20T14:55:10,233 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,44997,1732114507984/1a15ecfd95f4%2C44997%2C1732114507984.meta.1732114510214.meta 2024-11-20T14:55:10,233 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T14:55:10,234 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T14:55:10,235 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T14:55:10,238 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T14:55:10,242 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T14:55:10,245 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T14:55:10,246 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:10,246 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T14:55:10,246 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T14:55:10,248 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T14:55:10,249 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T14:55:10,250 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:10,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:10,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T14:55:10,251 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T14:55:10,251 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:10,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:10,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T14:55:10,253 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T14:55:10,253 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:10,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:10,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T14:55:10,254 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T14:55:10,255 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:10,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:10,255 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T14:55:10,257 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740 2024-11-20T14:55:10,258 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740 2024-11-20T14:55:10,258 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T14:55:10,258 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T14:55:10,258 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T14:55:10,259 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T14:55:10,260 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67702380, jitterRate=0.008844077587127686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T14:55:10,260 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T14:55:10,262 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732114510247Writing region info on filesystem at 1732114510247Initializing all the Stores at 1732114510247Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114510247Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114510248 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114510248Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114510248Cleaning up temporary data from old regions at 1732114510258 (+10 ms)Running coprocessor post-open hooks at 1732114510260 (+2 ms)Region opened successfully at 1732114510262 (+2 ms) 2024-11-20T14:55:10,268 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732114510189 2024-11-20T14:55:10,278 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T14:55:10,278 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T14:55:10,279 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:10,280 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,44997,1732114507984, state=OPEN 2024-11-20T14:55:10,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T14:55:10,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T14:55:10,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T14:55:10,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:10,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:10,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:10,305 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:10,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T14:55:10,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,44997,1732114507984 in 283 msec 2024-11-20T14:55:10,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T14:55:10,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 774 msec 2024-11-20T14:55:10,316 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T14:55:10,316 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T14:55:10,335 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T14:55:10,337 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a15ecfd95f4,44997,1732114507984, seqNum=-1] 2024-11-20T14:55:10,357 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T14:55:10,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46877, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T14:55:10,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0820 sec 2024-11-20T14:55:10,382 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732114510382, completionTime=-1 2024-11-20T14:55:10,384 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T14:55:10,384 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T14:55:10,408 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T14:55:10,408 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732114570408 2024-11-20T14:55:10,408 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732114630408 2024-11-20T14:55:10,408 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-20T14:55:10,410 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T14:55:10,415 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,45021,1732114507133-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:10,416 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,45021,1732114507133-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:10,416 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,45021,1732114507133-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:10,417 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a15ecfd95f4:45021, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:10,418 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:10,418 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:10,424 DEBUG [master/1a15ecfd95f4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T14:55:10,443 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.096sec 2024-11-20T14:55:10,445 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T14:55:10,446 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T14:55:10,447 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T14:55:10,447 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T14:55:10,447 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T14:55:10,448 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,45021,1732114507133-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T14:55:10,449 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,45021,1732114507133-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T14:55:10,450 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T14:55:10,451 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T14:55:10,451 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,45021,1732114507133-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:10,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885e29b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T14:55:10,552 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T14:55:10,552 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T14:55:10,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a15ecfd95f4,45021,-1 for getting cluster id 2024-11-20T14:55:10,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T14:55:10,568 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9158d560-6048-49a1-9ba6-ee08dcbba61e' 2024-11-20T14:55:10,570 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T14:55:10,571 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9158d560-6048-49a1-9ba6-ee08dcbba61e" 2024-11-20T14:55:10,571 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cab19f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T14:55:10,571 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a15ecfd95f4,45021,-1] 2024-11-20T14:55:10,574 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T14:55:10,576 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:10,577 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T14:55:10,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1714ff62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T14:55:10,581 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T14:55:10,588 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a15ecfd95f4,44997,1732114507984, seqNum=-1] 2024-11-20T14:55:10,589 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T14:55:10,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T14:55:10,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1a15ecfd95f4,45021,1732114507133 2024-11-20T14:55:10,616 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T14:55:10,622 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a15ecfd95f4,45021,1732114507133 2024-11-20T14:55:10,624 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fff83fd 2024-11-20T14:55:10,625 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T14:55:10,629 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T14:55:10,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732114510611', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T14:55:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732114510611 2024-11-20T14:55:10,657 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732114510611 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T14:55:10,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732114510611" procId is: 4 2024-11-20T14:55:10,660 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:10,662 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732114510611 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T14:55:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T14:55:10,679 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732114510611-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fbc196f3671a11da94b28abe6cc208a2, NAME => 'testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732114510611', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 2024-11-20T14:55:10,685 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732114510611-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:10,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732114510611-pool-0 {}] regionserver.HRegion(1722): Closing fbc196f3671a11da94b28abe6cc208a2, disabling compactions & flushes 2024-11-20T14:55:10,686 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732114510611-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:10,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732114510611-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:10,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732114510611-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. after waiting 0 ms 2024-11-20T14:55:10,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732114510611-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:10,686 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732114510611-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:10,686 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732114510611-pool-0 {}] regionserver.HRegion(1676): Region close journal for fbc196f3671a11da94b28abe6cc208a2: Waiting for close lock at 1732114510686Disabling compacts and flushes for region at 1732114510686Disabling writes for close at 1732114510686Writing region close event to WAL at 1732114510686Closed at 1732114510686 2024-11-20T14:55:10,687 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732114510611 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T14:55:10,692 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732114510688"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732114510688"}]},"ts":"1732114510688"} 2024-11-20T14:55:10,696 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T14:55:10,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732114510611 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T14:55:10,700 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732114510611","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732114510697"}]},"ts":"1732114510697"} 2024-11-20T14:55:10,704 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732114510611, state=ENABLING in hbase:meta 2024-11-20T14:55:10,705 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {1a15ecfd95f4=0} racks are {/default-rack=0} 2024-11-20T14:55:10,706 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T14:55:10,706 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T14:55:10,706 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T14:55:10,706 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T14:55:10,706 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T14:55:10,706 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T14:55:10,706 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T14:55:10,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, ASSIGN}] 2024-11-20T14:55:10,711 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, ASSIGN 2024-11-20T14:55:10,712 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,44165,1732114508156; forceNewPlan=false, retain=false 2024-11-20T14:55:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T14:55:10,866 INFO [1a15ecfd95f4:45021 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T14:55:10,867 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fbc196f3671a11da94b28abe6cc208a2, regionState=OPENING, regionLocation=1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:10,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, ASSIGN because future has completed 2024-11-20T14:55:10,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbc196f3671a11da94b28abe6cc208a2, server=1a15ecfd95f4,44165,1732114508156}] 2024-11-20T14:55:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T14:55:11,028 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T14:55:11,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41389, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T14:55:11,032 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:11,032 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fbc196f3671a11da94b28abe6cc208a2, NAME => 'testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T14:55:11,033 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732114510611 fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,033 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:11,033 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,033 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,034 INFO [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,037 INFO [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc196f3671a11da94b28abe6cc208a2 columnFamilyName family 2024-11-20T14:55:11,038 DEBUG [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:11,038 INFO [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] regionserver.HStore(327): Store=fbc196f3671a11da94b28abe6cc208a2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:11,038 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,039 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,039 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,040 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,040 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,041 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,046 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T14:55:11,047 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fbc196f3671a11da94b28abe6cc208a2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69221198, jitterRate=0.031476229429244995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T14:55:11,047 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:11,048 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fbc196f3671a11da94b28abe6cc208a2: Running coprocessor pre-open hook at 1732114511033Writing region info on filesystem at 1732114511033Initializing all the Stores at 1732114511034 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114511034Cleaning up temporary data from old regions at 1732114511040 (+6 ms)Running coprocessor post-open hooks at 1732114511047 (+7 ms)Region opened successfully at 1732114511048 (+1 ms) 2024-11-20T14:55:11,049 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2., pid=6, masterSystemTime=1732114511028 2024-11-20T14:55:11,052 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:11,052 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:11,053 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fbc196f3671a11da94b28abe6cc208a2, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:11,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbc196f3671a11da94b28abe6cc208a2, server=1a15ecfd95f4,44165,1732114508156 because future has completed 2024-11-20T14:55:11,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T14:55:11,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fbc196f3671a11da94b28abe6cc208a2, server=1a15ecfd95f4,44165,1732114508156 in 184 msec 2024-11-20T14:55:11,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T14:55:11,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, ASSIGN in 352 msec 2024-11-20T14:55:11,063 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732114510611 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T14:55:11,063 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732114510611","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732114511063"}]},"ts":"1732114511063"} 2024-11-20T14:55:11,066 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732114510611, state=ENABLED in hbase:meta 2024-11-20T14:55:11,067 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732114510611 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T14:55:11,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732114510611 in 417 msec 2024-11-20T14:55:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T14:55:11,296 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732114510611 get assigned. Timeout = 60000ms 2024-11-20T14:55:11,296 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732114510611 completed 2024-11-20T14:55:11,300 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T14:55:11,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732114510611 assigned to meta. Checking AM states. 2024-11-20T14:55:11,306 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T14:55:11,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732114510611 assigned. 2024-11-20T14:55:11,315 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732114510611', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2., hostname=1a15ecfd95f4,44165,1732114508156, seqNum=2] 2024-11-20T14:55:11,317 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T14:55:11,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T14:55:11,329 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fbc196f3671a11da94b28abe6cc208a2 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T14:55:11,352 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 20b144e1718a4c8799df8e799a8dd9a6 2024-11-20T14:55:11,391 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/.tmp/family/20b144e1718a4c8799df8e799a8dd9a6 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T14:55:11,395 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/.tmp/family/20b144e1718a4c8799df8e799a8dd9a6 2024-11-20T14:55:11,428 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/.tmp/family/20b144e1718a4c8799df8e799a8dd9a6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6 2024-11-20T14:55:11,430 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6, delay=978 ms 2024-11-20T14:55:11,431 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T14:55:11,437 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for fbc196f3671a11da94b28abe6cc208a2 in 109ms, sequenceid=6, compaction requested=false 2024-11-20T14:55:11,438 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732114510611' 2024-11-20T14:55:11,441 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fbc196f3671a11da94b28abe6cc208a2: 2024-11-20T14:55:12,413 DEBUG [hfile-prefetch-1732114511430 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 20b144e1718a4c8799df8e799a8dd9a6_0. 2024-11-20T14:55:12,420 DEBUG [hfile-prefetch-1732114511430 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 20b144e1718a4c8799df8e799a8dd9a6_117. 2024-11-20T14:55:12,421 DEBUG [hfile-prefetch-1732114511430 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6, with total blocks 2, and data blocks 1 2024-11-20T14:55:12,421 DEBUG [hfile-prefetch-1732114511430 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T14:55:12,421 DEBUG [hfile-prefetch-1732114511430 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T14:55:12,421 DEBUG [hfile-prefetch-1732114511430 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6. Will sleep 100ms and try the verification again. 2024-11-20T14:55:12,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=fbc196f3671a11da94b28abe6cc208a2, source=1a15ecfd95f4,44165,1732114508156, destination=1a15ecfd95f4,44997,1732114507984, warming up region on 1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:12,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=fbc196f3671a11da94b28abe6cc208a2, source=1a15ecfd95f4,44165,1732114508156, destination=1a15ecfd95f4,44997,1732114507984, running balancer 2024-11-20T14:55:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, REOPEN/MOVE 2024-11-20T14:55:12,460 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, REOPEN/MOVE 2024-11-20T14:55:12,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(7855): Warmup {ENCODED => fbc196f3671a11da94b28abe6cc208a2, NAME => 'testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T14:55:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:12,462 INFO [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,462 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fbc196f3671a11da94b28abe6cc208a2, regionState=CLOSING, regionLocation=1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:12,463 INFO [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc196f3671a11da94b28abe6cc208a2 columnFamilyName family 2024-11-20T14:55:12,463 DEBUG [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:12,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, REOPEN/MOVE because future has completed 2024-11-20T14:55:12,467 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-20T14:55:12,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure fbc196f3671a11da94b28abe6cc208a2, server=1a15ecfd95f4,44165,1732114508156}] 2024-11-20T14:55:12,468 DEBUG [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6 2024-11-20T14:55:12,469 INFO [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] regionserver.HStore(327): Store=fbc196f3671a11da94b28abe6cc208a2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(1722): Closing fbc196f3671a11da94b28abe6cc208a2, disabling compactions & flushes 2024-11-20T14:55:12,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. after waiting 0 ms 2024-11-20T14:55:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,471 DEBUG [StoreFileCloser-fbc196f3671a11da94b28abe6cc208a2-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6 2024-11-20T14:55:12,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(1676): Region close journal for fbc196f3671a11da94b28abe6cc208a2: Waiting for close lock at 1732114512469Disabling compacts and flushes for region at 1732114512469Disabling writes for close at 1732114512469Writing region close event to WAL at 1732114512471 (+2 ms)Closed at 1732114512471 2024-11-20T14:55:12,473 WARN [hfile-prefetch-1732114511430 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-20T14:55:12,474 DEBUG [hfile-prefetch-1732114511430 {}] hfile.PrefetchExecutor(121): Prefetch completed for 20b144e1718a4c8799df8e799a8dd9a6 2024-11-20T14:55:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45021 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T14:55:12,514 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T14:55:12,514 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T14:55:12,515 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T14:55:12,515 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T14:55:12,540 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-20T14:55:12,540 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732114512514 2024-11-20T14:55:12,540 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T14:55:12,625 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,626 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-20T14:55:12,626 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing fbc196f3671a11da94b28abe6cc208a2, disabling compactions & flushes 2024-11-20T14:55:12,626 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,626 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,626 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. after waiting 0 ms 2024-11-20T14:55:12,626 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,628 DEBUG [StoreFileCloser-fbc196f3671a11da94b28abe6cc208a2-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 20b144e1718a4c8799df8e799a8dd9a6, starting offset: 0, end offset: 9223372036854775807 2024-11-20T14:55:12,629 DEBUG [StoreFileCloser-fbc196f3671a11da94b28abe6cc208a2-family-1 {}] bucket.BucketCache(856): removed key 20b144e1718a4c8799df8e799a8dd9a6_0 from back map with offset lock 2094080 in the evict process 2024-11-20T14:55:12,633 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T14:55:12,635 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,635 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for fbc196f3671a11da94b28abe6cc208a2: Waiting for close lock at 1732114512626Running coprocessor pre-close hooks at 1732114512626Disabling compacts and flushes for region at 1732114512626Disabling writes for close at 1732114512626Writing region close event to WAL at 1732114512629 (+3 ms)Running coprocessor post-close hooks at 1732114512634 (+5 ms)Closed at 1732114512635 (+1 ms) 2024-11-20T14:55:12,635 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding fbc196f3671a11da94b28abe6cc208a2 move to 1a15ecfd95f4,44997,1732114507984 record at close sequenceid=6 2024-11-20T14:55:12,639 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,640 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fbc196f3671a11da94b28abe6cc208a2, regionState=CLOSED 2024-11-20T14:55:12,641 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T14:55:12,641 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T14:55:12,641 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T14:55:12,641 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T14:55:12,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure fbc196f3671a11da94b28abe6cc208a2, server=1a15ecfd95f4,44165,1732114508156 because future has completed 2024-11-20T14:55:12,645 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T14:55:12,645 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732114512641 2024-11-20T14:55:12,645 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T14:55:12,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T14:55:12,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure fbc196f3671a11da94b28abe6cc208a2, server=1a15ecfd95f4,44165,1732114508156 in 177 msec 2024-11-20T14:55:12,649 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, REOPEN/MOVE; state=CLOSED, location=1a15ecfd95f4,44997,1732114507984; forceNewPlan=false, retain=false 2024-11-20T14:55:12,799 INFO [1a15ecfd95f4:45021 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T14:55:12,800 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fbc196f3671a11da94b28abe6cc208a2, regionState=OPENING, regionLocation=1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:12,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, REOPEN/MOVE because future has completed 2024-11-20T14:55:12,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbc196f3671a11da94b28abe6cc208a2, server=1a15ecfd95f4,44997,1732114507984}] 2024-11-20T14:55:12,961 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,961 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => fbc196f3671a11da94b28abe6cc208a2, NAME => 'testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T14:55:12,962 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732114510611 fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,962 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:12,962 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,962 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,963 INFO [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,965 INFO [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc196f3671a11da94b28abe6cc208a2 columnFamilyName family 2024-11-20T14:55:12,965 DEBUG [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:12,966 DEBUG [StoreFileOpener-fbc196f3671a11da94b28abe6cc208a2-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6, delay=924 ms 2024-11-20T14:55:12,968 DEBUG [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6 2024-11-20T14:55:12,968 INFO [StoreOpener-fbc196f3671a11da94b28abe6cc208a2-1 {}] regionserver.HStore(327): Store=fbc196f3671a11da94b28abe6cc208a2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:12,968 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,969 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,970 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,970 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,970 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,974 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,975 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened fbc196f3671a11da94b28abe6cc208a2; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61851156, jitterRate=-0.07834595441818237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T14:55:12,975 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:12,976 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for fbc196f3671a11da94b28abe6cc208a2: Running coprocessor pre-open hook at 1732114512963Writing region info on filesystem at 1732114512963Initializing all the Stores at 1732114512963Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114512963Cleaning up temporary data from old regions at 1732114512970 (+7 ms)Running coprocessor post-open hooks at 1732114512975 (+5 ms)Region opened successfully at 1732114512976 (+1 ms) 2024-11-20T14:55:12,977 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2., pid=9, masterSystemTime=1732114512957 2024-11-20T14:55:12,980 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,980 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:12,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fbc196f3671a11da94b28abe6cc208a2, regionState=OPEN, openSeqNum=10, regionLocation=1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:12,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbc196f3671a11da94b28abe6cc208a2, server=1a15ecfd95f4,44997,1732114507984 because future has completed 2024-11-20T14:55:12,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T14:55:12,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure fbc196f3671a11da94b28abe6cc208a2, server=1a15ecfd95f4,44997,1732114507984 in 180 msec 2024-11-20T14:55:12,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732114510611, region=fbc196f3671a11da94b28abe6cc208a2, REOPEN/MOVE in 529 msec 2024-11-20T14:55:13,013 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T14:55:13,014 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T14:55:13,014 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T14:55:13,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:13,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T14:55:13,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:13,022 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T14:55:13,022 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-20T14:55:13,023 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1a15ecfd95f4,45021,1732114507133 2024-11-20T14:55:13,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:13,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:13,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:13,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:13,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:13,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:13,078 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T14:55:13,080 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T14:55:13,080 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T14:55:13,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:13,081 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1a15ecfd95f4,44997,1732114507984' ***** 2024-11-20T14:55:13,081 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:13,081 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:13,081 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:13,081 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T14:55:13,081 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1a15ecfd95f4,44165,1732114508156' ***** 2024-11-20T14:55:13,082 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T14:55:13,082 INFO [RS:0;1a15ecfd95f4:44997 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T14:55:13,082 INFO [RS:1;1a15ecfd95f4:44165 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T14:55:13,082 INFO [RS:0;1a15ecfd95f4:44997 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/bucket.persistence 2024-11-20T14:55:13,082 INFO [RS:1;1a15ecfd95f4:44165 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/bucket.persistence 2024-11-20T14:55:13,083 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T14:55:13,083 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T14:55:13,083 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T14:55:13,083 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T14:55:13,083 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T14:55:13,083 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T14:55:13,083 INFO [RS:0;1a15ecfd95f4:44997 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T14:55:13,083 INFO [RS:1;1a15ecfd95f4:44165 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T14:55:13,083 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T14:55:13,083 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T14:55:13,093 DEBUG [RS:0;1a15ecfd95f4:44997 {}] bucket.BucketCache(1487): Thread RS:0;1a15ecfd95f4:44997 started persisting bucket cache to file 2024-11-20T14:55:13,093 DEBUG [RS:1;1a15ecfd95f4:44165 {}] bucket.BucketCache(1487): Thread RS:1;1a15ecfd95f4:44165 started persisting bucket cache to file 2024-11-20T14:55:13,093 DEBUG [RS:0;1a15ecfd95f4:44997 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T14:55:13,093 DEBUG [RS:0;1a15ecfd95f4:44997 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T14:55:13,093 DEBUG [RS:1;1a15ecfd95f4:44165 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T14:55:13,094 DEBUG [RS:1;1a15ecfd95f4:44165 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T14:55:13,097 DEBUG [RS:0;1a15ecfd95f4:44997 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T14:55:13,097 DEBUG [RS:0;1a15ecfd95f4:44997 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732114513093 2024-11-20T14:55:13,097 DEBUG [RS:0;1a15ecfd95f4:44997 {}] bucket.BucketCache(1509): Thread RS:0;1a15ecfd95f4:44997 finished persisting bucket cache to file, renaming 2024-11-20T14:55:13,097 DEBUG [RS:1;1a15ecfd95f4:44165 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T14:55:13,097 DEBUG [RS:1;1a15ecfd95f4:44165 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732114513093 2024-11-20T14:55:13,097 DEBUG [RS:1;1a15ecfd95f4:44165 {}] bucket.BucketCache(1509): Thread RS:1;1a15ecfd95f4:44165 finished persisting bucket cache to file, renaming 2024-11-20T14:55:13,097 WARN [RS:1;1a15ecfd95f4:44165 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-20T14:55:13,097 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T14:55:13,098 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T14:55:13,098 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T14:55:13,098 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T14:55:13,098 INFO [RS:1;1a15ecfd95f4:44165 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T14:55:13,098 INFO [RS:0;1a15ecfd95f4:44997 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T14:55:13,098 INFO [RS:1;1a15ecfd95f4:44165 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T14:55:13,098 INFO [RS:0;1a15ecfd95f4:44997 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T14:55:13,099 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(959): stopping server 1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:13,099 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(3091): Received CLOSE for fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:13,099 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T14:55:13,099 INFO [RS:1;1a15ecfd95f4:44165 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1a15ecfd95f4:44165. 2024-11-20T14:55:13,099 DEBUG [RS:1;1a15ecfd95f4:44165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T14:55:13,099 DEBUG [RS:1;1a15ecfd95f4:44165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:13,099 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(959): stopping server 1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:13,099 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T14:55:13,099 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(976): stopping server 1a15ecfd95f4,44165,1732114508156; all regions closed. 2024-11-20T14:55:13,099 INFO [RS:0;1a15ecfd95f4:44997 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1a15ecfd95f4:44997. 2024-11-20T14:55:13,100 DEBUG [RS:0;1a15ecfd95f4:44997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T14:55:13,100 DEBUG [RS:0;1a15ecfd95f4:44997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:13,100 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T14:55:13,100 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T14:55:13,100 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T14:55:13,100 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T14:55:13,100 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fbc196f3671a11da94b28abe6cc208a2, disabling compactions & flushes 2024-11-20T14:55:13,100 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T14:55:13,100 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:13,100 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:13,100 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. after waiting 0 ms 2024-11-20T14:55:13,100 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:13,101 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T14:55:13,101 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T14:55:13,101 DEBUG [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, fbc196f3671a11da94b28abe6cc208a2=testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2.} 2024-11-20T14:55:13,101 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T14:55:13,101 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T14:55:13,101 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T14:55:13,101 DEBUG [StoreFileCloser-fbc196f3671a11da94b28abe6cc208a2-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/family/20b144e1718a4c8799df8e799a8dd9a6 2024-11-20T14:55:13,101 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T14:55:13,101 DEBUG [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, fbc196f3671a11da94b28abe6cc208a2 2024-11-20T14:55:13,101 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T14:55:13,102 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-20T14:55:13,104 DEBUG [RS:1;1a15ecfd95f4:44165 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs 2024-11-20T14:55:13,105 INFO [RS:1;1a15ecfd95f4:44165 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a15ecfd95f4%2C44165%2C1732114508156:(num 1732114509904) 2024-11-20T14:55:13,105 DEBUG [RS:1;1a15ecfd95f4:44165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:13,105 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T14:55:13,105 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T14:55:13,105 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.ChoreService(370): Chore service for: regionserver/1a15ecfd95f4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T14:55:13,106 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T14:55:13,106 INFO [regionserver/1a15ecfd95f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T14:55:13,106 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T14:55:13,106 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T14:55:13,106 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T14:55:13,106 INFO [RS:1;1a15ecfd95f4:44165 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44165 2024-11-20T14:55:13,109 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnRegionMove1732114510611/fbc196f3671a11da94b28abe6cc208a2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T14:55:13,109 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:13,109 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fbc196f3671a11da94b28abe6cc208a2: Waiting for close lock at 1732114513099Running coprocessor pre-close hooks at 1732114513100 (+1 ms)Disabling compacts and flushes for region at 1732114513100Disabling writes for close at 1732114513100Writing region close event to WAL at 1732114513102 (+2 ms)Running coprocessor post-close hooks at 1732114513109 (+7 ms)Closed at 1732114513109 2024-11-20T14:55:13,109 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2. 2024-11-20T14:55:13,112 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e305b69a5f9b49fba55c2fa76b2060d0 2024-11-20T14:55:13,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a15ecfd95f4,44165,1732114508156 2024-11-20T14:55:13,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T14:55:13,124 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T14:55:13,131 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/info/e305b69a5f9b49fba55c2fa76b2060d0 is 205, key is testBlockEvictionOnRegionMove1732114510611,,1732114510631.fbc196f3671a11da94b28abe6cc208a2./info:regioninfo/1732114512980/Put/seqid=0 2024-11-20T14:55:13,132 INFO [regionserver/1a15ecfd95f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T14:55:13,132 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a15ecfd95f4,44165,1732114508156] 2024-11-20T14:55:13,132 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/info/e305b69a5f9b49fba55c2fa76b2060d0 2024-11-20T14:55:13,140 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1a15ecfd95f4,44165,1732114508156 already deleted, retry=false 2024-11-20T14:55:13,140 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1a15ecfd95f4,44165,1732114508156 expired; onlineServers=1 2024-11-20T14:55:13,142 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 96cd074e8d5f487fb4efb14bfe91a5c6 2024-11-20T14:55:13,150 INFO [regionserver/1a15ecfd95f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T14:55:13,156 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/ns/96cd074e8d5f487fb4efb14bfe91a5c6 is 43, key is default/ns:d/1732114510366/Put/seqid=0 2024-11-20T14:55:13,156 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/ns/96cd074e8d5f487fb4efb14bfe91a5c6 2024-11-20T14:55:13,165 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bc6275effdb647b8b6284732fd3b51e3 2024-11-20T14:55:13,177 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/table/bc6275effdb647b8b6284732fd3b51e3 is 78, key is testBlockEvictionOnRegionMove1732114510611/table:state/1732114511063/Put/seqid=0 2024-11-20T14:55:13,178 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/table/bc6275effdb647b8b6284732fd3b51e3 2024-11-20T14:55:13,181 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/info/e305b69a5f9b49fba55c2fa76b2060d0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/info/e305b69a5f9b49fba55c2fa76b2060d0 2024-11-20T14:55:13,183 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/info/e305b69a5f9b49fba55c2fa76b2060d0, delay=965 ms 2024-11-20T14:55:13,184 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/info/e305b69a5f9b49fba55c2fa76b2060d0, entries=15, sequenceid=15, filesize=7.6 K 2024-11-20T14:55:13,185 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/ns/96cd074e8d5f487fb4efb14bfe91a5c6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/ns/96cd074e8d5f487fb4efb14bfe91a5c6 2024-11-20T14:55:13,187 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/ns/96cd074e8d5f487fb4efb14bfe91a5c6, delay=925 ms 2024-11-20T14:55:13,189 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/ns/96cd074e8d5f487fb4efb14bfe91a5c6, entries=2, sequenceid=15, filesize=5.0 K 2024-11-20T14:55:13,189 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/table/bc6275effdb647b8b6284732fd3b51e3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/table/bc6275effdb647b8b6284732fd3b51e3 2024-11-20T14:55:13,191 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/table/bc6275effdb647b8b6284732fd3b51e3, delay=939 ms 2024-11-20T14:55:13,193 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/table/bc6275effdb647b8b6284732fd3b51e3, entries=2, sequenceid=15, filesize=5.3 K 2024-11-20T14:55:13,194 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false 2024-11-20T14:55:13,194 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T14:55:13,195 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/info/e305b69a5f9b49fba55c2fa76b2060d0 2024-11-20T14:55:13,196 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/ns/96cd074e8d5f487fb4efb14bfe91a5c6 2024-11-20T14:55:13,197 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/table/bc6275effdb647b8b6284732fd3b51e3 2024-11-20T14:55:13,201 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T14:55:13,201 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T14:55:13,202 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T14:55:13,202 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732114513101Running coprocessor pre-close hooks at 1732114513101Disabling compacts and flushes for region at 1732114513101Disabling writes for close at 1732114513101Obtaining lock to block concurrent updates at 1732114513102 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732114513102Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732114513102Flushing stores of hbase:meta,,1.1588230740 at 1732114513103 (+1 ms)Flushing 1588230740/info: creating writer at 1732114513103Flushing 1588230740/info: appending metadata at 1732114513130 (+27 ms)Flushing 1588230740/info: closing flushed file at 1732114513130Flushing 1588230740/ns: creating writer at 1732114513136 (+6 ms)Flushing 1588230740/ns: appending metadata at 1732114513155 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732114513155Flushing 1588230740/table: creating writer at 1732114513159 (+4 ms)Flushing 1588230740/table: appending metadata at 1732114513176 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732114513176Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29285fa0: reopening flushed file at 1732114513181 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e5aa6e5: reopening flushed file at 1732114513185 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fa155cb: reopening flushed file at 1732114513189 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false at 1732114513194 (+5 ms)Writing region close event to WAL at 1732114513197 (+3 ms)Running coprocessor post-close hooks at 1732114513201 (+4 ms)Closed at 1732114513202 (+1 ms) 2024-11-20T14:55:13,202 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T14:55:13,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:13,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44165-0x1015a0776f40002, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:13,233 INFO [RS:1;1a15ecfd95f4:44165 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T14:55:13,233 INFO [RS:1;1a15ecfd95f4:44165 {}] regionserver.HRegionServer(1031): Exiting; stopping=1a15ecfd95f4,44165,1732114508156; zookeeper connection closed. 2024-11-20T14:55:13,234 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a6588b2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a6588b2 2024-11-20T14:55:13,302 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(976): stopping server 1a15ecfd95f4,44997,1732114507984; all regions closed. 2024-11-20T14:55:13,305 DEBUG [RS:0;1a15ecfd95f4:44997 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs 2024-11-20T14:55:13,305 INFO [RS:0;1a15ecfd95f4:44997 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a15ecfd95f4%2C44997%2C1732114507984.meta:.meta(num 1732114510214) 2024-11-20T14:55:13,309 DEBUG [RS:0;1a15ecfd95f4:44997 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs 2024-11-20T14:55:13,309 INFO [RS:0;1a15ecfd95f4:44997 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a15ecfd95f4%2C44997%2C1732114507984:(num 1732114509904) 2024-11-20T14:55:13,309 DEBUG [RS:0;1a15ecfd95f4:44997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:13,309 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T14:55:13,310 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T14:55:13,310 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.ChoreService(370): Chore service for: regionserver/1a15ecfd95f4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T14:55:13,310 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T14:55:13,311 INFO [regionserver/1a15ecfd95f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T14:55:13,311 INFO [RS:0;1a15ecfd95f4:44997 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44997 2024-11-20T14:55:13,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a15ecfd95f4,44997,1732114507984 2024-11-20T14:55:13,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T14:55:13,323 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T14:55:13,324 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a15ecfd95f4,44997,1732114507984] 2024-11-20T14:55:13,340 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1a15ecfd95f4,44997,1732114507984 already deleted, retry=false 2024-11-20T14:55:13,340 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1a15ecfd95f4,44997,1732114507984 expired; onlineServers=0 2024-11-20T14:55:13,341 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1a15ecfd95f4,45021,1732114507133' ***** 2024-11-20T14:55:13,341 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T14:55:13,341 INFO [M:0;1a15ecfd95f4:45021 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T14:55:13,341 INFO [M:0;1a15ecfd95f4:45021 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T14:55:13,341 DEBUG [M:0;1a15ecfd95f4:45021 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T14:55:13,341 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T14:55:13,341 DEBUG [M:0;1a15ecfd95f4:45021 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T14:55:13,341 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.small.0-1732114509417 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.small.0-1732114509417,5,FailOnTimeoutGroup] 2024-11-20T14:55:13,341 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.large.0-1732114509414 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.large.0-1732114509414,5,FailOnTimeoutGroup] 2024-11-20T14:55:13,342 INFO [M:0;1a15ecfd95f4:45021 {}] hbase.ChoreService(370): Chore service for: master/1a15ecfd95f4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T14:55:13,342 INFO [M:0;1a15ecfd95f4:45021 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T14:55:13,342 DEBUG [M:0;1a15ecfd95f4:45021 {}] master.HMaster(1795): Stopping service threads 2024-11-20T14:55:13,342 INFO [M:0;1a15ecfd95f4:45021 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T14:55:13,342 INFO [M:0;1a15ecfd95f4:45021 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T14:55:13,343 INFO [M:0;1a15ecfd95f4:45021 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T14:55:13,343 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T14:55:13,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T14:55:13,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:13,353 DEBUG [M:0;1a15ecfd95f4:45021 {}] zookeeper.ZKUtil(347): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T14:55:13,353 WARN [M:0;1a15ecfd95f4:45021 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T14:55:13,354 INFO [M:0;1a15ecfd95f4:45021 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/.lastflushedseqids 2024-11-20T14:55:13,368 INFO [M:0;1a15ecfd95f4:45021 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T14:55:13,368 INFO [M:0;1a15ecfd95f4:45021 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T14:55:13,369 DEBUG [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T14:55:13,369 INFO [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:13,369 DEBUG [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:13,369 DEBUG [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T14:55:13,369 DEBUG [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:13,369 INFO [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-20T14:55:13,382 DEBUG [M:0;1a15ecfd95f4:45021 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f2cc4be9f9a942f8a711de3dd21368b0 2024-11-20T14:55:13,401 DEBUG [M:0;1a15ecfd95f4:45021 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f2cc4be9f9a942f8a711de3dd21368b0 is 82, key is hbase:meta,,1/info:regioninfo/1732114510278/Put/seqid=0 2024-11-20T14:55:13,402 INFO [M:0;1a15ecfd95f4:45021 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f2cc4be9f9a942f8a711de3dd21368b0 2024-11-20T14:55:13,410 DEBUG [M:0;1a15ecfd95f4:45021 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2ae6c765db4c406280fb0d2be86ee1ba 2024-11-20T14:55:13,429 DEBUG [M:0;1a15ecfd95f4:45021 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2ae6c765db4c406280fb0d2be86ee1ba is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732114511068/Put/seqid=0 2024-11-20T14:55:13,430 INFO [M:0;1a15ecfd95f4:45021 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2ae6c765db4c406280fb0d2be86ee1ba 2024-11-20T14:55:13,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:13,432 INFO [RS:0;1a15ecfd95f4:44997 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T14:55:13,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44997-0x1015a0776f40001, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:13,432 INFO [RS:0;1a15ecfd95f4:44997 {}] regionserver.HRegionServer(1031): Exiting; stopping=1a15ecfd95f4,44997,1732114507984; zookeeper connection closed. 2024-11-20T14:55:13,432 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39710de2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39710de2 2024-11-20T14:55:13,433 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-20T14:55:13,439 DEBUG [M:0;1a15ecfd95f4:45021 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b140715c67cd4c79ab327f89feb2f867 2024-11-20T14:55:13,456 DEBUG [M:0;1a15ecfd95f4:45021 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b140715c67cd4c79ab327f89feb2f867 is 69, key is 1a15ecfd95f4,44165,1732114508156/rs:state/1732114509506/Put/seqid=0 2024-11-20T14:55:13,457 INFO [M:0;1a15ecfd95f4:45021 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b140715c67cd4c79ab327f89feb2f867 2024-11-20T14:55:13,459 DEBUG [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f2cc4be9f9a942f8a711de3dd21368b0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f2cc4be9f9a942f8a711de3dd21368b0 2024-11-20T14:55:13,462 INFO [M:0;1a15ecfd95f4:45021 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f2cc4be9f9a942f8a711de3dd21368b0, entries=8, sequenceid=84, filesize=5.5 K 2024-11-20T14:55:13,463 DEBUG [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2ae6c765db4c406280fb0d2be86ee1ba as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2ae6c765db4c406280fb0d2be86ee1ba 2024-11-20T14:55:13,465 INFO [M:0;1a15ecfd95f4:45021 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2ae6c765db4c406280fb0d2be86ee1ba, entries=9, sequenceid=84, filesize=6.5 K 2024-11-20T14:55:13,466 DEBUG [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b140715c67cd4c79ab327f89feb2f867 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b140715c67cd4c79ab327f89feb2f867 2024-11-20T14:55:13,468 INFO [M:0;1a15ecfd95f4:45021 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b140715c67cd4c79ab327f89feb2f867, entries=2, sequenceid=84, filesize=5.1 K 2024-11-20T14:55:13,468 INFO [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 99ms, sequenceid=84, compaction requested=false 2024-11-20T14:55:13,470 INFO [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:13,470 DEBUG [M:0;1a15ecfd95f4:45021 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732114513369Disabling compacts and flushes for region at 1732114513369Disabling writes for close at 1732114513369Obtaining lock to block concurrent updates at 1732114513369Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732114513369Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732114513370 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732114513370Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732114513370Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732114513400 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732114513400Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732114513404 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732114513428 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732114513428Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732114513433 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732114513455 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732114513455Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21955de2: reopening flushed file at 1732114513459 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a11bb59: reopening flushed file at 1732114513462 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@277f6ec4: reopening flushed file at 1732114513465 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 99ms, sequenceid=84, compaction requested=false at 1732114513469 (+4 ms)Writing region close event to WAL at 1732114513470 (+1 ms)Closed at 1732114513470 2024-11-20T14:55:13,472 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T14:55:13,472 INFO [M:0;1a15ecfd95f4:45021 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T14:55:13,472 INFO [M:0;1a15ecfd95f4:45021 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45021 2024-11-20T14:55:13,473 INFO [M:0;1a15ecfd95f4:45021 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T14:55:13,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:13,595 INFO [M:0;1a15ecfd95f4:45021 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T14:55:13,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45021-0x1015a0776f40000, quorum=127.0.0.1:63621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:13,605 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T14:55:13,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T14:55:13,624 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:13,632 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/1a15ecfd95f4:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/1a15ecfd95f4:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/1a15ecfd95f4:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732114511430 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732114512966 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/1a15ecfd95f4:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732114513183 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732114513187 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=381 (was 372) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10794 (was 11344) 2024-11-20T14:55:13,637 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=381, ProcessCount=11, AvailableMemoryMB=10793 2024-11-20T14:55:13,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/hadoop-log-dir so I do NOT create it in target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9 2024-11-20T14:55:13,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/hadoop-tmp-dir so I do NOT create it in target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9 2024-11-20T14:55:13,638 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:13,640 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/cluster_72d191d6-03fc-d272-20f5-5101ef902186, deleteOnExit=true 2024-11-20T14:55:13,657 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/cluster_72d191d6-03fc-d272-20f5-5101ef902186/zookeeper_0, clientPort=64993, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/cluster_72d191d6-03fc-d272-20f5-5101ef902186/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/cluster_72d191d6-03fc-d272-20f5-5101ef902186/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T14:55:13,658 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64993 2024-11-20T14:55:13,671 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 with version=8 2024-11-20T14:55:13,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/hbase-staging 2024-11-20T14:55:13,674 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T14:55:13,674 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:13,674 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:13,674 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T14:55:13,674 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:13,675 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T14:55:13,675 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T14:55:13,675 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T14:55:13,676 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37933 2024-11-20T14:55:13,678 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37933 connecting to ZooKeeper ensemble=127.0.0.1:64993 2024-11-20T14:55:13,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379330x0, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T14:55:13,717 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37933-0x1015a0791bd0000 connected 2024-11-20T14:55:13,808 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:13,809 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:13,811 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:13,812 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8, hbase.cluster.distributed=false 2024-11-20T14:55:13,817 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T14:55:13,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37933 2024-11-20T14:55:13,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37933 2024-11-20T14:55:13,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37933 2024-11-20T14:55:13,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37933 2024-11-20T14:55:13,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37933 2024-11-20T14:55:13,836 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T14:55:13,836 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:13,836 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:13,836 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T14:55:13,836 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:13,836 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T14:55:13,836 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T14:55:13,836 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T14:55:13,837 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33835 2024-11-20T14:55:13,838 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33835 connecting to ZooKeeper ensemble=127.0.0.1:64993 2024-11-20T14:55:13,839 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:13,839 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:13,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338350x0, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T14:55:13,850 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33835-0x1015a0791bd0001 connected 2024-11-20T14:55:13,850 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:13,850 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T14:55:13,851 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.cache 2024-11-20T14:55:13,851 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T14:55:13,851 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T14:55:13,851 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T14:55:13,852 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T14:55:13,852 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T14:55:13,852 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T14:55:13,852 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T14:55:13,853 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T14:55:13,853 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.persistence is complete. 2024-11-20T14:55:13,853 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T14:55:13,855 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T14:55:13,855 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33835 2024-11-20T14:55:13,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33835 2024-11-20T14:55:13,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33835 2024-11-20T14:55:13,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33835 2024-11-20T14:55:13,857 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33835 2024-11-20T14:55:13,879 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T14:55:13,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:13,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:13,879 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T14:55:13,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:13,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T14:55:13,879 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T14:55:13,879 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T14:55:13,880 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38417 2024-11-20T14:55:13,882 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38417 connecting to ZooKeeper ensemble=127.0.0.1:64993 2024-11-20T14:55:13,883 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:13,883 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:13,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:384170x0, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T14:55:13,896 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38417-0x1015a0791bd0002 connected 2024-11-20T14:55:13,896 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:13,896 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T14:55:13,897 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.cache 2024-11-20T14:55:13,897 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T14:55:13,897 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T14:55:13,897 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T14:55:13,898 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T14:55:13,898 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T14:55:13,898 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T14:55:13,898 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.persistence is complete. 2024-11-20T14:55:13,898 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T14:55:13,900 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T14:55:13,901 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T14:55:13,902 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T14:55:13,903 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38417 2024-11-20T14:55:13,904 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38417 2024-11-20T14:55:13,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38417 2024-11-20T14:55:13,907 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38417 2024-11-20T14:55:13,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38417 2024-11-20T14:55:13,921 DEBUG [M:0;1a15ecfd95f4:37933 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a15ecfd95f4:37933 2024-11-20T14:55:13,921 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:13,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:13,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:13,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:13,933 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:13,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T14:55:13,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:13,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T14:55:13,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:13,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:13,947 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T14:55:13,948 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a15ecfd95f4,37933,1732114513673 from backup master directory 2024-11-20T14:55:13,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:13,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:13,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:13,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T14:55:13,958 WARN [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T14:55:13,958 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:13,960 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/hbase.id] with ID: b33f4785-59a9-4e66-a575-30b490b0825e 2024-11-20T14:55:13,960 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/.tmp/hbase.id 2024-11-20T14:55:13,966 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/hbase.id] 2024-11-20T14:55:14,001 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:14,001 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T14:55:14,003 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-20T14:55:14,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,045 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T14:55:14,047 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T14:55:14,047 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T14:55:14,069 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store 2024-11-20T14:55:14,078 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:14,078 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T14:55:14,078 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:14,078 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:14,078 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T14:55:14,078 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:14,078 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:14,078 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732114514078Disabling compacts and flushes for region at 1732114514078Disabling writes for close at 1732114514078Writing region close event to WAL at 1732114514078Closed at 1732114514078 2024-11-20T14:55:14,079 WARN [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/.initializing 2024-11-20T14:55:14,079 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/WALs/1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:14,086 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T14:55:14,090 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a15ecfd95f4%2C37933%2C1732114513673, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/WALs/1a15ecfd95f4,37933,1732114513673, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/oldWALs, maxLogs=10 2024-11-20T14:55:14,108 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/WALs/1a15ecfd95f4,37933,1732114513673/1a15ecfd95f4%2C37933%2C1732114513673.1732114514090 2024-11-20T14:55:14,108 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T14:55:14,108 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T14:55:14,109 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:14,109 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,109 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T14:55:14,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:14,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T14:55:14,118 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:14,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T14:55:14,121 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:14,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T14:55:14,125 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:14,125 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,126 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,126 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,126 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,126 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,126 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T14:55:14,127 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T14:55:14,132 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T14:55:14,132 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72182527, jitterRate=0.07560347020626068}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T14:55:14,133 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732114514109Initializing all the Stores at 1732114514109Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114514109Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114514109Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114514109Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114514109Cleaning up temporary data from old regions at 1732114514126 (+17 ms)Region opened successfully at 1732114514133 (+7 ms) 2024-11-20T14:55:14,133 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T14:55:14,137 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5423f95a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T14:55:14,138 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T14:55:14,138 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T14:55:14,138 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T14:55:14,138 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T14:55:14,138 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T14:55:14,139 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-20T14:55:14,139 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T14:55:14,140 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T14:55:14,141 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T14:55:14,166 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T14:55:14,166 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T14:55:14,167 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T14:55:14,178 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T14:55:14,179 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T14:55:14,180 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T14:55:14,190 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T14:55:14,192 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T14:55:14,199 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T14:55:14,201 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T14:55:14,212 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T14:55:14,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:14,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:14,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:14,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,221 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1a15ecfd95f4,37933,1732114513673, sessionid=0x1015a0791bd0000, setting cluster-up flag (Was=false) 2024-11-20T14:55:14,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,266 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T14:55:14,267 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:14,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,312 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T14:55:14,313 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:14,314 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T14:55:14,315 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T14:55:14,316 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T14:55:14,316 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T14:55:14,316 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a15ecfd95f4,37933,1732114513673 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T14:55:14,318 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T14:55:14,318 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T14:55:14,318 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T14:55:14,318 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T14:55:14,318 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a15ecfd95f4:0, corePoolSize=10, maxPoolSize=10 2024-11-20T14:55:14,318 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,318 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T14:55:14,319 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,319 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732114544319 2024-11-20T14:55:14,320 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T14:55:14,320 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T14:55:14,320 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T14:55:14,320 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T14:55:14,320 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T14:55:14,320 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T14:55:14,320 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T14:55:14,320 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,320 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T14:55:14,320 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T14:55:14,320 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T14:55:14,321 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T14:55:14,321 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,321 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T14:55:14,321 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T14:55:14,321 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T14:55:14,321 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.large.0-1732114514321,5,FailOnTimeoutGroup] 2024-11-20T14:55:14,321 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.small.0-1732114514321,5,FailOnTimeoutGroup] 2024-11-20T14:55:14,321 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,321 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T14:55:14,322 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,322 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,338 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T14:55:14,338 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 2024-11-20T14:55:14,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:14,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T14:55:14,348 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T14:55:14,348 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:14,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T14:55:14,352 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T14:55:14,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:14,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T14:55:14,357 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T14:55:14,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:14,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T14:55:14,361 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T14:55:14,361 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:14,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T14:55:14,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740 2024-11-20T14:55:14,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740 2024-11-20T14:55:14,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T14:55:14,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T14:55:14,363 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T14:55:14,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T14:55:14,369 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T14:55:14,369 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73633768, jitterRate=0.09722864627838135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T14:55:14,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732114514344Initializing all the Stores at 1732114514344Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114514345 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114514345Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114514345Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114514345Cleaning up temporary data from old regions at 1732114514363 (+18 ms)Region opened successfully at 1732114514370 (+7 ms) 2024-11-20T14:55:14,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T14:55:14,370 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T14:55:14,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T14:55:14,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T14:55:14,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T14:55:14,371 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T14:55:14,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732114514370Disabling compacts and flushes for region at 1732114514370Disabling writes for close at 1732114514370Writing region close event to WAL at 1732114514371 (+1 ms)Closed at 1732114514371 2024-11-20T14:55:14,372 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T14:55:14,372 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T14:55:14,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T14:55:14,373 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T14:55:14,374 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T14:55:14,412 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(746): ClusterId : b33f4785-59a9-4e66-a575-30b490b0825e 2024-11-20T14:55:14,412 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(746): ClusterId : b33f4785-59a9-4e66-a575-30b490b0825e 2024-11-20T14:55:14,412 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T14:55:14,412 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T14:55:14,446 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T14:55:14,446 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T14:55:14,446 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T14:55:14,446 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T14:55:14,459 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T14:55:14,459 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T14:55:14,460 DEBUG [RS:1;1a15ecfd95f4:38417 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cb293b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T14:55:14,460 DEBUG [RS:0;1a15ecfd95f4:33835 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@274f6e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T14:55:14,482 DEBUG [RS:0;1a15ecfd95f4:33835 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a15ecfd95f4:33835 2024-11-20T14:55:14,482 DEBUG [RS:1;1a15ecfd95f4:38417 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1a15ecfd95f4:38417 2024-11-20T14:55:14,482 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T14:55:14,482 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T14:55:14,482 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T14:55:14,482 DEBUG [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T14:55:14,482 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T14:55:14,482 DEBUG [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T14:55:14,483 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a15ecfd95f4,37933,1732114513673 with port=33835, startcode=1732114513835 2024-11-20T14:55:14,483 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a15ecfd95f4,37933,1732114513673 with port=38417, startcode=1732114513878 2024-11-20T14:55:14,483 DEBUG [RS:0;1a15ecfd95f4:33835 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T14:55:14,483 DEBUG [RS:1;1a15ecfd95f4:38417 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T14:55:14,491 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33991, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T14:55:14,492 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44609, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T14:55:14,492 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37933 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:14,492 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37933 {}] master.ServerManager(517): Registering regionserver=1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:14,494 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37933 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1a15ecfd95f4,33835,1732114513835 2024-11-20T14:55:14,494 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37933 {}] master.ServerManager(517): Registering regionserver=1a15ecfd95f4,33835,1732114513835 2024-11-20T14:55:14,495 DEBUG [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 2024-11-20T14:55:14,495 DEBUG [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T14:55:14,495 DEBUG [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T14:55:14,496 DEBUG [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 2024-11-20T14:55:14,496 DEBUG [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T14:55:14,496 DEBUG [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T14:55:14,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T14:55:14,524 WARN [1a15ecfd95f4:37933 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T14:55:14,534 DEBUG [RS:1;1a15ecfd95f4:38417 {}] zookeeper.ZKUtil(111): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:14,534 WARN [RS:1;1a15ecfd95f4:38417 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T14:55:14,534 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a15ecfd95f4,38417,1732114513878] 2024-11-20T14:55:14,534 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a15ecfd95f4,33835,1732114513835] 2024-11-20T14:55:14,534 INFO [RS:1;1a15ecfd95f4:38417 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T14:55:14,534 DEBUG [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:14,534 DEBUG [RS:0;1a15ecfd95f4:33835 {}] zookeeper.ZKUtil(111): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a15ecfd95f4,33835,1732114513835 2024-11-20T14:55:14,534 WARN [RS:0;1a15ecfd95f4:33835 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T14:55:14,534 INFO [RS:0;1a15ecfd95f4:33835 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T14:55:14,535 DEBUG [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,33835,1732114513835 2024-11-20T14:55:14,541 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T14:55:14,541 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T14:55:14,543 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T14:55:14,545 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T14:55:14,545 INFO [RS:1;1a15ecfd95f4:38417 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T14:55:14,545 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,546 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T14:55:14,546 INFO [RS:0;1a15ecfd95f4:33835 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T14:55:14,546 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,546 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T14:55:14,546 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T14:55:14,547 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,547 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T14:55:14,547 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,547 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,547 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,547 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,547 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,547 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,547 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,547 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T14:55:14,548 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T14:55:14,548 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,548 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T14:55:14,549 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:14,549 DEBUG [RS:0;1a15ecfd95f4:33835 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T14:55:14,549 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T14:55:14,549 DEBUG [RS:1;1a15ecfd95f4:38417 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T14:55:14,549 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,33835,1732114513835-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,550 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,38417,1732114513878-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T14:55:14,567 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T14:55:14,567 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,33835,1732114513835-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,568 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,568 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.Replication(171): 1a15ecfd95f4,33835,1732114513835 started 2024-11-20T14:55:14,568 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T14:55:14,568 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,38417,1732114513878-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,568 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,568 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.Replication(171): 1a15ecfd95f4,38417,1732114513878 started 2024-11-20T14:55:14,587 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,587 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(1482): Serving as 1a15ecfd95f4,33835,1732114513835, RpcServer on 1a15ecfd95f4/172.17.0.2:33835, sessionid=0x1015a0791bd0001 2024-11-20T14:55:14,587 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T14:55:14,587 DEBUG [RS:0;1a15ecfd95f4:33835 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a15ecfd95f4,33835,1732114513835 2024-11-20T14:55:14,587 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,33835,1732114513835' 2024-11-20T14:55:14,587 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T14:55:14,588 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T14:55:14,588 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:14,589 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(1482): Serving as 1a15ecfd95f4,38417,1732114513878, RpcServer on 1a15ecfd95f4/172.17.0.2:38417, sessionid=0x1015a0791bd0002 2024-11-20T14:55:14,589 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T14:55:14,589 DEBUG [RS:1;1a15ecfd95f4:38417 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:14,589 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,38417,1732114513878' 2024-11-20T14:55:14,589 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T14:55:14,589 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T14:55:14,589 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T14:55:14,589 DEBUG [RS:0;1a15ecfd95f4:33835 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a15ecfd95f4,33835,1732114513835 2024-11-20T14:55:14,589 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,33835,1732114513835' 2024-11-20T14:55:14,589 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T14:55:14,590 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T14:55:14,590 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T14:55:14,590 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T14:55:14,590 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T14:55:14,590 DEBUG [RS:1;1a15ecfd95f4:38417 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:14,590 DEBUG [RS:0;1a15ecfd95f4:33835 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T14:55:14,590 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,38417,1732114513878' 2024-11-20T14:55:14,590 INFO [RS:0;1a15ecfd95f4:33835 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T14:55:14,590 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T14:55:14,590 INFO [RS:0;1a15ecfd95f4:33835 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T14:55:14,591 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T14:55:14,591 DEBUG [RS:1;1a15ecfd95f4:38417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T14:55:14,591 INFO [RS:1;1a15ecfd95f4:38417 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T14:55:14,591 INFO [RS:1;1a15ecfd95f4:38417 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T14:55:14,691 INFO [RS:0;1a15ecfd95f4:33835 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T14:55:14,692 INFO [RS:0;1a15ecfd95f4:33835 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1a15ecfd95f4%2C33835%2C1732114513835, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,33835,1732114513835, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs, maxLogs=55 2024-11-20T14:55:14,692 INFO [RS:1;1a15ecfd95f4:38417 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T14:55:14,693 INFO [RS:1;1a15ecfd95f4:38417 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1a15ecfd95f4%2C38417%2C1732114513878, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,38417,1732114513878, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs, maxLogs=55 2024-11-20T14:55:14,723 INFO [RS:0;1a15ecfd95f4:33835 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,33835,1732114513835/1a15ecfd95f4%2C33835%2C1732114513835.1732114514695 2024-11-20T14:55:14,723 INFO [RS:1;1a15ecfd95f4:38417 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,38417,1732114513878/1a15ecfd95f4%2C38417%2C1732114513878.1732114514696 2024-11-20T14:55:14,724 DEBUG [RS:0;1a15ecfd95f4:33835 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T14:55:14,724 DEBUG [RS:1;1a15ecfd95f4:38417 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T14:55:14,774 DEBUG [1a15ecfd95f4:37933 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T14:55:14,775 DEBUG [1a15ecfd95f4:37933 {}] balancer.BalancerClusterState(204): Hosts are {1a15ecfd95f4=0} racks are {/default-rack=0} 2024-11-20T14:55:14,776 DEBUG [1a15ecfd95f4:37933 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T14:55:14,776 DEBUG [1a15ecfd95f4:37933 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T14:55:14,776 DEBUG [1a15ecfd95f4:37933 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T14:55:14,776 DEBUG [1a15ecfd95f4:37933 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T14:55:14,776 INFO [1a15ecfd95f4:37933 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T14:55:14,776 INFO [1a15ecfd95f4:37933 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T14:55:14,776 DEBUG [1a15ecfd95f4:37933 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T14:55:14,776 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:14,778 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,38417,1732114513878, state=OPENING 2024-11-20T14:55:14,799 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T14:55:14,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:14,808 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T14:55:14,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:14,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,38417,1732114513878}] 2024-11-20T14:55:14,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:14,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:14,962 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T14:55:14,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60541, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T14:55:14,966 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T14:55:14,966 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T14:55:14,967 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T14:55:14,967 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1a15ecfd95f4%2C38417%2C1732114513878.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,38417,1732114513878, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs, maxLogs=55 2024-11-20T14:55:14,987 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,38417,1732114513878/1a15ecfd95f4%2C38417%2C1732114513878.meta.1732114514968.meta 2024-11-20T14:55:14,988 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T14:55:14,988 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T14:55:14,989 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T14:55:14,989 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T14:55:14,989 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T14:55:14,989 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T14:55:14,989 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:14,989 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T14:55:14,989 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T14:55:14,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T14:55:14,991 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T14:55:14,992 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:14,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T14:55:14,993 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T14:55:14,993 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:14,994 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T14:55:14,994 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T14:55:14,994 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:14,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T14:55:14,995 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T14:55:14,996 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:14,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T14:55:14,996 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T14:55:14,997 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740 2024-11-20T14:55:14,998 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740 2024-11-20T14:55:14,998 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T14:55:14,998 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T14:55:14,998 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T14:55:14,999 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T14:55:15,000 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61510739, jitterRate=-0.08341856300830841}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T14:55:15,000 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T14:55:15,001 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732114514990Writing region info on filesystem at 1732114514990Initializing all the Stores at 1732114514990Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114514990Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114514990Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114514991 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732114514991Cleaning up temporary data from old regions at 1732114514998 (+7 ms)Running coprocessor post-open hooks at 1732114515000 (+2 ms)Region opened successfully at 1732114515001 (+1 ms) 2024-11-20T14:55:15,002 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732114514961 2024-11-20T14:55:15,005 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T14:55:15,005 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T14:55:15,006 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:15,007 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,38417,1732114513878, state=OPEN 2024-11-20T14:55:15,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T14:55:15,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T14:55:15,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T14:55:15,037 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:15,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:15,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:15,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T14:55:15,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T14:55:15,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,38417,1732114513878 in 229 msec 2024-11-20T14:55:15,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T14:55:15,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 668 msec 2024-11-20T14:55:15,043 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T14:55:15,043 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T14:55:15,047 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T14:55:15,047 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a15ecfd95f4,38417,1732114513878, seqNum=-1] 2024-11-20T14:55:15,048 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T14:55:15,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52431, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T14:55:15,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 741 msec 2024-11-20T14:55:15,057 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732114515057, completionTime=-1 2024-11-20T14:55:15,057 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T14:55:15,058 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T14:55:15,061 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T14:55:15,061 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732114575061 2024-11-20T14:55:15,061 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732114635061 2024-11-20T14:55:15,061 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-20T14:55:15,062 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T14:55:15,063 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,37933,1732114513673-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:15,063 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,37933,1732114513673-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:15,063 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,37933,1732114513673-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:15,063 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a15ecfd95f4:37933, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:15,063 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:15,064 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:15,068 DEBUG [master/1a15ecfd95f4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T14:55:15,070 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.112sec 2024-11-20T14:55:15,071 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T14:55:15,071 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T14:55:15,071 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T14:55:15,071 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T14:55:15,071 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T14:55:15,071 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,37933,1732114513673-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T14:55:15,071 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,37933,1732114513673-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T14:55:15,072 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T14:55:15,072 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T14:55:15,072 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,37933,1732114513673-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:15,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5858d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T14:55:15,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a15ecfd95f4,37933,-1 for getting cluster id 2024-11-20T14:55:15,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T14:55:15,116 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b33f4785-59a9-4e66-a575-30b490b0825e' 2024-11-20T14:55:15,117 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T14:55:15,117 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b33f4785-59a9-4e66-a575-30b490b0825e" 2024-11-20T14:55:15,118 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3488a1dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T14:55:15,118 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a15ecfd95f4,37933,-1] 2024-11-20T14:55:15,119 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T14:55:15,119 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:15,121 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T14:55:15,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@717481fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T14:55:15,123 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T14:55:15,125 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a15ecfd95f4,38417,1732114513878, seqNum=-1] 2024-11-20T14:55:15,125 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T14:55:15,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T14:55:15,134 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:15,134 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T14:55:15,136 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:15,136 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d9cd7a8 2024-11-20T14:55:15,137 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T14:55:15,139 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T14:55:15,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37933 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732114515134', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T14:55:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37933 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732114515134 2024-11-20T14:55:15,143 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732114515134 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T14:55:15,143 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:15,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37933 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732114515134" procId is: 4 2024-11-20T14:55:15,144 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732114515134 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T14:55:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T14:55:15,156 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732114515134-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b4e5d9ba3b9003858db8605ee521fb01, NAME => 'testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732114515134', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 2024-11-20T14:55:15,163 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732114515134-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:15,163 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732114515134-pool-0 {}] regionserver.HRegion(1722): Closing b4e5d9ba3b9003858db8605ee521fb01, disabling compactions & flushes 2024-11-20T14:55:15,163 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732114515134-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:15,163 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732114515134-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:15,163 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732114515134-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. after waiting 0 ms 2024-11-20T14:55:15,164 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732114515134-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:15,164 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732114515134-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:15,164 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732114515134-pool-0 {}] regionserver.HRegion(1676): Region close journal for b4e5d9ba3b9003858db8605ee521fb01: Waiting for close lock at 1732114515163Disabling compacts and flushes for region at 1732114515163Disabling writes for close at 1732114515164 (+1 ms)Writing region close event to WAL at 1732114515164Closed at 1732114515164 2024-11-20T14:55:15,165 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732114515134 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T14:55:15,166 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732114515165"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732114515165"}]},"ts":"1732114515165"} 2024-11-20T14:55:15,169 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T14:55:15,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732114515134 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T14:55:15,170 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732114515134","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732114515170"}]},"ts":"1732114515170"} 2024-11-20T14:55:15,174 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732114515134, state=ENABLING in hbase:meta 2024-11-20T14:55:15,174 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {1a15ecfd95f4=0} racks are {/default-rack=0} 2024-11-20T14:55:15,175 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T14:55:15,175 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T14:55:15,175 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T14:55:15,175 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T14:55:15,175 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T14:55:15,175 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T14:55:15,175 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T14:55:15,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732114515134, region=b4e5d9ba3b9003858db8605ee521fb01, ASSIGN}] 2024-11-20T14:55:15,177 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732114515134, region=b4e5d9ba3b9003858db8605ee521fb01, ASSIGN 2024-11-20T14:55:15,178 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732114515134, region=b4e5d9ba3b9003858db8605ee521fb01, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,33835,1732114513835; forceNewPlan=false, retain=false 2024-11-20T14:55:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T14:55:15,329 INFO [1a15ecfd95f4:37933 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T14:55:15,329 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b4e5d9ba3b9003858db8605ee521fb01, regionState=OPENING, regionLocation=1a15ecfd95f4,33835,1732114513835 2024-11-20T14:55:15,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732114515134, region=b4e5d9ba3b9003858db8605ee521fb01, ASSIGN because future has completed 2024-11-20T14:55:15,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4e5d9ba3b9003858db8605ee521fb01, server=1a15ecfd95f4,33835,1732114513835}] 2024-11-20T14:55:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T14:55:15,487 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T14:55:15,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52577, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T14:55:15,492 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:15,492 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b4e5d9ba3b9003858db8605ee521fb01, NAME => 'testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01.', STARTKEY => '', ENDKEY => ''} 2024-11-20T14:55:15,493 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732114515134 b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,493 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:15,493 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,493 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,494 INFO [StoreOpener-b4e5d9ba3b9003858db8605ee521fb01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,498 INFO [StoreOpener-b4e5d9ba3b9003858db8605ee521fb01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4e5d9ba3b9003858db8605ee521fb01 columnFamilyName family 2024-11-20T14:55:15,498 DEBUG [StoreOpener-b4e5d9ba3b9003858db8605ee521fb01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:15,499 INFO [StoreOpener-b4e5d9ba3b9003858db8605ee521fb01-1 {}] regionserver.HStore(327): Store=b4e5d9ba3b9003858db8605ee521fb01/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:15,499 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,499 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,500 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,500 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,500 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,500 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,509 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T14:55:15,510 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b4e5d9ba3b9003858db8605ee521fb01; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67506658, jitterRate=0.005927592515945435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T14:55:15,510 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:15,511 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b4e5d9ba3b9003858db8605ee521fb01: Running coprocessor pre-open hook at 1732114515493Writing region info on filesystem at 1732114515493Initializing all the Stores at 1732114515493Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114515493Cleaning up temporary data from old regions at 1732114515500 (+7 ms)Running coprocessor post-open hooks at 1732114515510 (+10 ms)Region opened successfully at 1732114515511 (+1 ms) 2024-11-20T14:55:15,512 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01., pid=6, masterSystemTime=1732114515486 2024-11-20T14:55:15,515 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:15,515 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:15,516 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b4e5d9ba3b9003858db8605ee521fb01, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,33835,1732114513835 2024-11-20T14:55:15,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4e5d9ba3b9003858db8605ee521fb01, server=1a15ecfd95f4,33835,1732114513835 because future has completed 2024-11-20T14:55:15,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T14:55:15,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b4e5d9ba3b9003858db8605ee521fb01, server=1a15ecfd95f4,33835,1732114513835 in 188 msec 2024-11-20T14:55:15,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T14:55:15,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732114515134, region=b4e5d9ba3b9003858db8605ee521fb01, ASSIGN in 348 msec 2024-11-20T14:55:15,527 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732114515134 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T14:55:15,527 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732114515134","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732114515527"}]},"ts":"1732114515527"} 2024-11-20T14:55:15,530 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732114515134, state=ENABLED in hbase:meta 2024-11-20T14:55:15,532 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732114515134 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T14:55:15,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732114515134 in 391 msec 2024-11-20T14:55:15,748 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T14:55:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37933 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T14:55:15,776 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732114515134 completed 2024-11-20T14:55:15,776 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732114515134 get assigned. Timeout = 60000ms 2024-11-20T14:55:15,777 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T14:55:15,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732114515134 assigned to meta. Checking AM states. 2024-11-20T14:55:15,781 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T14:55:15,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732114515134 assigned. 2024-11-20T14:55:15,787 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732114515134', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01., hostname=1a15ecfd95f4,33835,1732114513835, seqNum=2] 2024-11-20T14:55:15,787 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T14:55:15,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T14:55:15,796 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b4e5d9ba3b9003858db8605ee521fb01 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T14:55:15,808 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 09bba72bdb2a40c282455ad3544a6c05 2024-11-20T14:55:15,823 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/.tmp/family/09bba72bdb2a40c282455ad3544a6c05 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T14:55:15,824 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/.tmp/family/09bba72bdb2a40c282455ad3544a6c05 2024-11-20T14:55:15,827 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/.tmp/family/09bba72bdb2a40c282455ad3544a6c05 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05 2024-11-20T14:55:15,828 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05, delay=954 ms 2024-11-20T14:55:15,830 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T14:55:15,831 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for b4e5d9ba3b9003858db8605ee521fb01 in 35ms, sequenceid=6, compaction requested=false 2024-11-20T14:55:15,832 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732114515134' 2024-11-20T14:55:15,833 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b4e5d9ba3b9003858db8605ee521fb01: 2024-11-20T14:55:16,784 DEBUG [hfile-prefetch-1732114512966 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 09bba72bdb2a40c282455ad3544a6c05_0. 2024-11-20T14:55:16,784 DEBUG [hfile-prefetch-1732114512966 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 09bba72bdb2a40c282455ad3544a6c05_117. 2024-11-20T14:55:16,785 DEBUG [hfile-prefetch-1732114512966 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05, with total blocks 2, and data blocks 1 2024-11-20T14:55:16,785 DEBUG [hfile-prefetch-1732114512966 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T14:55:16,785 DEBUG [hfile-prefetch-1732114512966 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T14:55:16,785 DEBUG [hfile-prefetch-1732114512966 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05. Will sleep 100ms and try the verification again. 2024-11-20T14:55:16,833 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;1a15ecfd95f4:33835,5,FailOnTimeoutGroup] 2024-11-20T14:55:16,833 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1a15ecfd95f4,33835,1732114513835' ***** 2024-11-20T14:55:16,834 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-20T14:55:16,834 INFO [RS:0;1a15ecfd95f4:33835 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T14:55:16,834 INFO [RS:0;1a15ecfd95f4:33835 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.persistence 2024-11-20T14:55:16,834 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T14:55:16,834 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T14:55:16,834 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T14:55:16,834 INFO [RS:0;1a15ecfd95f4:33835 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T14:55:16,834 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T14:55:16,844 DEBUG [RS:0;1a15ecfd95f4:33835 {}] bucket.BucketCache(1487): Thread RS:0;1a15ecfd95f4:33835 started persisting bucket cache to file 2024-11-20T14:55:16,845 DEBUG [RS:0;1a15ecfd95f4:33835 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T14:55:16,845 DEBUG [RS:0;1a15ecfd95f4:33835 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T14:55:16,848 DEBUG [RS:0;1a15ecfd95f4:33835 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-20T14:55:16,848 DEBUG [RS:0;1a15ecfd95f4:33835 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732114516845 2024-11-20T14:55:16,848 DEBUG [RS:0;1a15ecfd95f4:33835 {}] bucket.BucketCache(1509): Thread RS:0;1a15ecfd95f4:33835 finished persisting bucket cache to file, renaming 2024-11-20T14:55:16,848 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T14:55:16,848 INFO [RS:0;1a15ecfd95f4:33835 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T14:55:16,848 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T14:55:16,849 INFO [RS:0;1a15ecfd95f4:33835 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T14:55:16,849 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(3091): Received CLOSE for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:16,849 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(959): stopping server 1a15ecfd95f4,33835,1732114513835 2024-11-20T14:55:16,849 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T14:55:16,849 INFO [RS:0;1a15ecfd95f4:33835 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1a15ecfd95f4:33835. 2024-11-20T14:55:16,849 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T14:55:16,849 DEBUG [RS:0;1a15ecfd95f4:33835 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T14:55:16,849 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b4e5d9ba3b9003858db8605ee521fb01, disabling compactions & flushes 2024-11-20T14:55:16,849 DEBUG [RS:0;1a15ecfd95f4:33835 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:16,849 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:16,849 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:16,849 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T14:55:16,849 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. after waiting 0 ms 2024-11-20T14:55:16,849 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:16,849 DEBUG [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(1325): Online Regions={b4e5d9ba3b9003858db8605ee521fb01=testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01.} 2024-11-20T14:55:16,850 DEBUG [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(1351): Waiting on b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:16,850 DEBUG [StoreFileCloser-b4e5d9ba3b9003858db8605ee521fb01-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05 2024-11-20T14:55:16,850 WARN [hfile-prefetch-1732114512966 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-20T14:55:16,851 DEBUG [hfile-prefetch-1732114512966 {}] hfile.PrefetchExecutor(121): Prefetch completed for 09bba72bdb2a40c282455ad3544a6c05 2024-11-20T14:55:16,852 INFO [regionserver/1a15ecfd95f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T14:55:16,855 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T14:55:16,855 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:16,855 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b4e5d9ba3b9003858db8605ee521fb01: Waiting for close lock at 1732114516849Running coprocessor pre-close hooks at 1732114516849Disabling compacts and flushes for region at 1732114516849Disabling writes for close at 1732114516849Writing region close event to WAL at 1732114516850 (+1 ms)Running coprocessor post-close hooks at 1732114516855 (+5 ms)Closed at 1732114516855 2024-11-20T14:55:16,856 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:16,900 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T14:55:16,900 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T14:55:16,900 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T14:55:16,900 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T14:55:16,903 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T14:55:16,903 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732114516900 2024-11-20T14:55:16,903 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T14:55:17,050 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(976): stopping server 1a15ecfd95f4,33835,1732114513835; all regions closed. 2024-11-20T14:55:17,052 DEBUG [RS:0;1a15ecfd95f4:33835 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs 2024-11-20T14:55:17,052 INFO [RS:0;1a15ecfd95f4:33835 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a15ecfd95f4%2C33835%2C1732114513835:(num 1732114514695) 2024-11-20T14:55:17,052 DEBUG [RS:0;1a15ecfd95f4:33835 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:17,052 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T14:55:17,052 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T14:55:17,052 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.ChoreService(370): Chore service for: regionserver/1a15ecfd95f4:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T14:55:17,052 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T14:55:17,052 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T14:55:17,052 INFO [regionserver/1a15ecfd95f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T14:55:17,052 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T14:55:17,052 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T14:55:17,053 INFO [RS:0;1a15ecfd95f4:33835 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33835 2024-11-20T14:55:17,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T14:55:17,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a15ecfd95f4,33835,1732114513835 2024-11-20T14:55:17,082 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T14:55:17,084 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a15ecfd95f4,33835,1732114513835] 2024-11-20T14:55:17,098 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1a15ecfd95f4,33835,1732114513835 already deleted, retry=false 2024-11-20T14:55:17,098 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 1a15ecfd95f4,33835,1732114513835 on 1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:17,103 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 1a15ecfd95f4,33835,1732114513835, splitWal=true, meta=false 2024-11-20T14:55:17,106 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 1a15ecfd95f4,33835,1732114513835, splitWal=true, meta=false 2024-11-20T14:55:17,107 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 1a15ecfd95f4,33835,1732114513835 (carryingMeta=false) 1a15ecfd95f4,33835,1732114513835/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@60e03fff[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T14:55:17,108 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 1a15ecfd95f4,33835,1732114513835 had 1 regions 2024-11-20T14:55:17,109 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 1a15ecfd95f4,33835,1732114513835, splitWal=true, meta=false, isMeta: false 2024-11-20T14:55:17,110 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,33835,1732114513835-splitting 2024-11-20T14:55:17,110 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,33835,1732114513835-splitting dir is empty, no logs to split. 2024-11-20T14:55:17,110 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1a15ecfd95f4,33835,1732114513835 WAL count=0, meta=false 2024-11-20T14:55:17,112 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,33835,1732114513835-splitting dir is empty, no logs to split. 2024-11-20T14:55:17,112 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1a15ecfd95f4,33835,1732114513835 WAL count=0, meta=false 2024-11-20T14:55:17,112 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 1a15ecfd95f4,33835,1732114513835 WAL splitting is done? wals=0, meta=false 2024-11-20T14:55:17,112 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 1a15ecfd95f4,33835,1732114513835 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,33835,1732114513835-splitting does not exist 2024-11-20T14:55:17,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732114515134, region=b4e5d9ba3b9003858db8605ee521fb01, ASSIGN}] 2024-11-20T14:55:17,115 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732114515134, region=b4e5d9ba3b9003858db8605ee521fb01, ASSIGN 2024-11-20T14:55:17,116 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732114515134, region=b4e5d9ba3b9003858db8605ee521fb01, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T14:55:17,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:17,191 INFO [RS:0;1a15ecfd95f4:33835 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T14:55:17,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33835-0x1015a0791bd0001, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:17,191 INFO [RS:0;1a15ecfd95f4:33835 {}] regionserver.HRegionServer(1031): Exiting; stopping=1a15ecfd95f4,33835,1732114513835; zookeeper connection closed. 2024-11-20T14:55:17,191 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a066be6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a066be6 2024-11-20T14:55:17,267 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=b4e5d9ba3b9003858db8605ee521fb01, regionState=OPENING, regionLocation=1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:17,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732114515134, region=b4e5d9ba3b9003858db8605ee521fb01, ASSIGN because future has completed 2024-11-20T14:55:17,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4e5d9ba3b9003858db8605ee521fb01, server=1a15ecfd95f4,38417,1732114513878}] 2024-11-20T14:55:17,351 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T14:55:17,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:17,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:17,352 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T14:55:17,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T14:55:17,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T14:55:17,352 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T14:55:17,352 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T14:55:17,353 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39727 2024-11-20T14:55:17,355 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39727 connecting to ZooKeeper ensemble=127.0.0.1:64993 2024-11-20T14:55:17,355 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:17,355 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:17,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:397270x0, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T14:55:17,382 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39727-0x1015a0791bd0003 connected 2024-11-20T14:55:17,382 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39727-0x1015a0791bd0003, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-20T14:55:17,382 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-20T14:55:17,383 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T14:55:17,384 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.cache 2024-11-20T14:55:17,384 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T14:55:17,384 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T14:55:17,384 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T14:55:17,387 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T14:55:17,387 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T14:55:17,387 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-20T14:55:17,388 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T14:55:17,388 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-20T14:55:17,389 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39727-0x1015a0791bd0003, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T14:55:17,391 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39727-0x1015a0791bd0003, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T14:55:17,399 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-20T14:55:17,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39727 2024-11-20T14:55:17,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39727 2024-11-20T14:55:17,399 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T14:55:17,400 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-20T14:55:17,400 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.persistence is complete. 2024-11-20T14:55:17,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39727 2024-11-20T14:55:17,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39727 2024-11-20T14:55:17,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39727 2024-11-20T14:55:17,404 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(746): ClusterId : b33f4785-59a9-4e66-a575-30b490b0825e 2024-11-20T14:55:17,404 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T14:55:17,416 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T14:55:17,416 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T14:55:17,424 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T14:55:17,425 DEBUG [RS:2;1a15ecfd95f4:39727 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b833e35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T14:55:17,432 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:17,433 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => b4e5d9ba3b9003858db8605ee521fb01, NAME => 'testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01.', STARTKEY => '', ENDKEY => ''} 2024-11-20T14:55:17,433 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732114515134 b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,433 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T14:55:17,433 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,433 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,434 INFO [StoreOpener-b4e5d9ba3b9003858db8605ee521fb01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,435 INFO [StoreOpener-b4e5d9ba3b9003858db8605ee521fb01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4e5d9ba3b9003858db8605ee521fb01 columnFamilyName family 2024-11-20T14:55:17,435 DEBUG [StoreOpener-b4e5d9ba3b9003858db8605ee521fb01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T14:55:17,436 DEBUG [RS:2;1a15ecfd95f4:39727 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;1a15ecfd95f4:39727 2024-11-20T14:55:17,436 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T14:55:17,436 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T14:55:17,437 DEBUG [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T14:55:17,437 DEBUG [StoreFileOpener-b4e5d9ba3b9003858db8605ee521fb01-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05, delay=916 ms 2024-11-20T14:55:17,438 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a15ecfd95f4,37933,1732114513673 with port=39727, startcode=1732114517351 2024-11-20T14:55:17,438 DEBUG [RS:2;1a15ecfd95f4:39727 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T14:55:17,439 DEBUG [StoreOpener-b4e5d9ba3b9003858db8605ee521fb01-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05 2024-11-20T14:55:17,439 INFO [StoreOpener-b4e5d9ba3b9003858db8605ee521fb01-1 {}] regionserver.HStore(327): Store=b4e5d9ba3b9003858db8605ee521fb01/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T14:55:17,439 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,440 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,440 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37675, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T14:55:17,441 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,441 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37933 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1a15ecfd95f4,39727,1732114517351 2024-11-20T14:55:17,441 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,441 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37933 {}] master.ServerManager(517): Registering regionserver=1a15ecfd95f4,39727,1732114517351 2024-11-20T14:55:17,441 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,442 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,442 DEBUG [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8 2024-11-20T14:55:17,442 DEBUG [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T14:55:17,442 DEBUG [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T14:55:17,443 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened b4e5d9ba3b9003858db8605ee521fb01; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63199104, jitterRate=-0.05825996398925781}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T14:55:17,443 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:17,444 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for b4e5d9ba3b9003858db8605ee521fb01: Running coprocessor pre-open hook at 1732114517434Writing region info on filesystem at 1732114517434Initializing all the Stores at 1732114517434Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732114517434Cleaning up temporary data from old regions at 1732114517441 (+7 ms)Running coprocessor post-open hooks at 1732114517443 (+2 ms)Region opened successfully at 1732114517444 (+1 ms) 2024-11-20T14:55:17,445 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01., pid=9, masterSystemTime=1732114517430 2024-11-20T14:55:17,447 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:17,447 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:17,447 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=b4e5d9ba3b9003858db8605ee521fb01, regionState=OPEN, openSeqNum=10, regionLocation=1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:17,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4e5d9ba3b9003858db8605ee521fb01, server=1a15ecfd95f4,38417,1732114513878 because future has completed 2024-11-20T14:55:17,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T14:55:17,454 DEBUG [RS:2;1a15ecfd95f4:39727 {}] zookeeper.ZKUtil(111): regionserver:39727-0x1015a0791bd0003, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a15ecfd95f4,39727,1732114517351 2024-11-20T14:55:17,454 WARN [RS:2;1a15ecfd95f4:39727 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T14:55:17,454 INFO [RS:2;1a15ecfd95f4:39727 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T14:55:17,454 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a15ecfd95f4,39727,1732114517351] 2024-11-20T14:55:17,454 DEBUG [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,39727,1732114517351 2024-11-20T14:55:17,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-20T14:55:17,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure b4e5d9ba3b9003858db8605ee521fb01, server=1a15ecfd95f4,38417,1732114513878 in 177 msec 2024-11-20T14:55:17,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T14:55:17,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732114515134, region=b4e5d9ba3b9003858db8605ee521fb01, ASSIGN in 341 msec 2024-11-20T14:55:17,457 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 1a15ecfd95f4,33835,1732114513835 after splitting done 2024-11-20T14:55:17,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 1a15ecfd95f4,33835,1732114513835, splitWal=true, meta=false in 358 msec 2024-11-20T14:55:17,461 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T14:55:17,463 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T14:55:17,463 INFO [RS:2;1a15ecfd95f4:39727 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T14:55:17,463 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:17,464 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T14:55:17,464 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T14:55:17,464 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:17,464 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,464 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T14:55:17,465 DEBUG [RS:2;1a15ecfd95f4:39727 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T14:55:17,466 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:17,466 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:17,466 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:17,466 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:17,466 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:17,466 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,39727,1732114517351-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T14:55:17,482 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T14:55:17,482 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,39727,1732114517351-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:17,482 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:17,482 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.Replication(171): 1a15ecfd95f4,39727,1732114517351 started 2024-11-20T14:55:17,496 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T14:55:17,496 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(1482): Serving as 1a15ecfd95f4,39727,1732114517351, RpcServer on 1a15ecfd95f4/172.17.0.2:39727, sessionid=0x1015a0791bd0003 2024-11-20T14:55:17,497 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T14:55:17,497 DEBUG [RS:2;1a15ecfd95f4:39727 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a15ecfd95f4,39727,1732114517351 2024-11-20T14:55:17,497 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,39727,1732114517351' 2024-11-20T14:55:17,497 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T14:55:17,498 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T14:55:17,498 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T14:55:17,498 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T14:55:17,498 DEBUG [RS:2;1a15ecfd95f4:39727 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a15ecfd95f4,39727,1732114517351 2024-11-20T14:55:17,498 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,39727,1732114517351' 2024-11-20T14:55:17,498 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T14:55:17,499 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T14:55:17,500 DEBUG [RS:2;1a15ecfd95f4:39727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T14:55:17,500 INFO [RS:2;1a15ecfd95f4:39727 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T14:55:17,500 INFO [RS:2;1a15ecfd95f4:39727 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T14:55:17,600 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T14:55:17,601 INFO [RS:2;1a15ecfd95f4:39727 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T14:55:17,601 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T14:55:17,602 INFO [RS:2;1a15ecfd95f4:39727 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1a15ecfd95f4%2C39727%2C1732114517351, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,39727,1732114517351, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs, maxLogs=55 2024-11-20T14:55:17,606 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732114515134 2024-11-20T14:55:17,606 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732114515134 Metrics about Tables on a single HBase RegionServer 2024-11-20T14:55:17,608 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T14:55:17,608 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T14:55:17,609 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T14:55:17,609 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T14:55:17,622 INFO [RS:2;1a15ecfd95f4:39727 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/WALs/1a15ecfd95f4,39727,1732114517351/1a15ecfd95f4%2C39727%2C1732114517351.1732114517603 2024-11-20T14:55:17,622 DEBUG [RS:2;1a15ecfd95f4:39727 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T14:55:17,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T14:55:17,997 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T14:55:17,997 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T14:55:17,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:17,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:17,997 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T14:55:17,997 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T14:55:17,997 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=945284187, stopped=false 2024-11-20T14:55:17,997 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1a15ecfd95f4,37933,1732114513673 2024-11-20T14:55:18,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:18,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:18,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1015a0791bd0003, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T14:55:18,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:18,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:18,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1015a0791bd0003, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:18,053 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T14:55:18,053 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T14:55:18,054 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T14:55:18,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:18,054 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39727-0x1015a0791bd0003, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:18,054 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:18,054 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1a15ecfd95f4,38417,1732114513878' ***** 2024-11-20T14:55:18,054 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T14:55:18,054 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1a15ecfd95f4,39727,1732114517351' ***** 2024-11-20T14:55:18,054 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T14:55:18,054 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T14:55:18,054 INFO [RS:2;1a15ecfd95f4:39727 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T14:55:18,054 INFO [RS:2;1a15ecfd95f4:39727 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.persistence 2024-11-20T14:55:18,055 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T14:55:18,055 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T14:55:18,055 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T14:55:18,055 INFO [RS:1;1a15ecfd95f4:38417 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T14:55:18,055 INFO [RS:2;1a15ecfd95f4:39727 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T14:55:18,055 INFO [RS:1;1a15ecfd95f4:38417 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a5025fff-27b6-eef7-5034-e652b7c53de9/bucket.persistence 2024-11-20T14:55:18,055 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T14:55:18,055 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T14:55:18,055 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T14:55:18,055 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T14:55:18,055 INFO [RS:1;1a15ecfd95f4:38417 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T14:55:18,055 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T14:55:18,065 DEBUG [RS:2;1a15ecfd95f4:39727 {}] bucket.BucketCache(1487): Thread RS:2;1a15ecfd95f4:39727 started persisting bucket cache to file 2024-11-20T14:55:18,065 DEBUG [RS:2;1a15ecfd95f4:39727 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T14:55:18,065 DEBUG [RS:2;1a15ecfd95f4:39727 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T14:55:18,065 DEBUG [RS:1;1a15ecfd95f4:38417 {}] bucket.BucketCache(1487): Thread RS:1;1a15ecfd95f4:38417 started persisting bucket cache to file 2024-11-20T14:55:18,065 DEBUG [RS:1;1a15ecfd95f4:38417 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T14:55:18,066 DEBUG [RS:1;1a15ecfd95f4:38417 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T14:55:18,066 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T14:55:18,068 INFO [regionserver/1a15ecfd95f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T14:55:18,069 DEBUG [RS:2;1a15ecfd95f4:39727 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T14:55:18,069 DEBUG [RS:1;1a15ecfd95f4:38417 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T14:55:18,069 DEBUG [RS:2;1a15ecfd95f4:39727 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732114518065 2024-11-20T14:55:18,069 DEBUG [RS:1;1a15ecfd95f4:38417 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732114518065 2024-11-20T14:55:18,069 DEBUG [RS:2;1a15ecfd95f4:39727 {}] bucket.BucketCache(1509): Thread RS:2;1a15ecfd95f4:39727 finished persisting bucket cache to file, renaming 2024-11-20T14:55:18,069 DEBUG [RS:1;1a15ecfd95f4:38417 {}] bucket.BucketCache(1509): Thread RS:1;1a15ecfd95f4:38417 finished persisting bucket cache to file, renaming 2024-11-20T14:55:18,069 WARN [RS:1;1a15ecfd95f4:38417 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-20T14:55:18,069 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T14:55:18,069 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T14:55:18,069 INFO [RS:2;1a15ecfd95f4:39727 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T14:55:18,069 INFO [RS:1;1a15ecfd95f4:38417 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T14:55:18,069 INFO [RS:2;1a15ecfd95f4:39727 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T14:55:18,069 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T14:55:18,069 INFO [RS:1;1a15ecfd95f4:38417 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T14:55:18,069 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(959): stopping server 1a15ecfd95f4,39727,1732114517351 2024-11-20T14:55:18,069 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T14:55:18,069 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(3091): Received CLOSE for b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:18,070 INFO [RS:2;1a15ecfd95f4:39727 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;1a15ecfd95f4:39727. 2024-11-20T14:55:18,070 DEBUG [RS:2;1a15ecfd95f4:39727 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T14:55:18,070 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(959): stopping server 1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:18,070 DEBUG [RS:2;1a15ecfd95f4:39727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:18,070 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T14:55:18,070 INFO [RS:1;1a15ecfd95f4:38417 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1a15ecfd95f4:38417. 2024-11-20T14:55:18,070 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(976): stopping server 1a15ecfd95f4,39727,1732114517351; all regions closed. 2024-11-20T14:55:18,070 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T14:55:18,070 DEBUG [RS:1;1a15ecfd95f4:38417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T14:55:18,070 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b4e5d9ba3b9003858db8605ee521fb01, disabling compactions & flushes 2024-11-20T14:55:18,070 DEBUG [RS:1;1a15ecfd95f4:38417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:18,070 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:18,070 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:18,070 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T14:55:18,070 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T14:55:18,070 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. after waiting 0 ms 2024-11-20T14:55:18,070 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T14:55:18,070 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:18,070 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T14:55:18,071 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T14:55:18,071 DEBUG [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, b4e5d9ba3b9003858db8605ee521fb01=testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01.} 2024-11-20T14:55:18,071 DEBUG [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b4e5d9ba3b9003858db8605ee521fb01 2024-11-20T14:55:18,071 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T14:55:18,071 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T14:55:18,071 DEBUG [StoreFileCloser-b4e5d9ba3b9003858db8605ee521fb01-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/family/09bba72bdb2a40c282455ad3544a6c05 2024-11-20T14:55:18,071 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T14:55:18,071 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T14:55:18,071 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T14:55:18,071 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T14:55:18,072 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-20T14:55:18,072 DEBUG [RS:2;1a15ecfd95f4:39727 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs 2024-11-20T14:55:18,073 INFO [RS:2;1a15ecfd95f4:39727 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a15ecfd95f4%2C39727%2C1732114517351:(num 1732114517603) 2024-11-20T14:55:18,073 DEBUG [RS:2;1a15ecfd95f4:39727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:18,073 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T14:55:18,073 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T14:55:18,073 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.ChoreService(370): Chore service for: regionserver/1a15ecfd95f4:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T14:55:18,073 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T14:55:18,073 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T14:55:18,073 INFO [regionserver/1a15ecfd95f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T14:55:18,073 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T14:55:18,073 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T14:55:18,073 INFO [RS:2;1a15ecfd95f4:39727 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39727 2024-11-20T14:55:18,075 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/default/testBlockEvictionOnGracefulStop1732114515134/b4e5d9ba3b9003858db8605ee521fb01/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T14:55:18,076 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:18,076 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b4e5d9ba3b9003858db8605ee521fb01: Waiting for close lock at 1732114518070Running coprocessor pre-close hooks at 1732114518070Disabling compacts and flushes for region at 1732114518070Disabling writes for close at 1732114518070Writing region close event to WAL at 1732114518071 (+1 ms)Running coprocessor post-close hooks at 1732114518076 (+5 ms)Closed at 1732114518076 2024-11-20T14:55:18,076 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01. 2024-11-20T14:55:18,080 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4b9dfca8b8c84b05950c6e2ead0e83fa 2024-11-20T14:55:18,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T14:55:18,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1015a0791bd0003, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a15ecfd95f4,39727,1732114517351 2024-11-20T14:55:18,089 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T14:55:18,093 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/info/4b9dfca8b8c84b05950c6e2ead0e83fa is 209, key is testBlockEvictionOnGracefulStop1732114515134,,1732114515140.b4e5d9ba3b9003858db8605ee521fb01./info:regioninfo/1732114517447/Put/seqid=0 2024-11-20T14:55:18,094 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/info/4b9dfca8b8c84b05950c6e2ead0e83fa 2024-11-20T14:55:18,101 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cf886fac23e84d78b90935864ca74986 2024-11-20T14:55:18,103 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a15ecfd95f4,39727,1732114517351] 2024-11-20T14:55:18,111 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1a15ecfd95f4,39727,1732114517351 already deleted, retry=false 2024-11-20T14:55:18,111 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1a15ecfd95f4,39727,1732114517351 expired; onlineServers=1 2024-11-20T14:55:18,115 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/ns/cf886fac23e84d78b90935864ca74986 is 43, key is default/ns:d/1732114515050/Put/seqid=0 2024-11-20T14:55:18,116 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/ns/cf886fac23e84d78b90935864ca74986 2024-11-20T14:55:18,123 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6c10f51b8c4f4ae697e09316eaacfd7d 2024-11-20T14:55:18,135 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/table/6c10f51b8c4f4ae697e09316eaacfd7d is 80, key is testBlockEvictionOnGracefulStop1732114515134/table:state/1732114515527/Put/seqid=0 2024-11-20T14:55:18,136 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/table/6c10f51b8c4f4ae697e09316eaacfd7d 2024-11-20T14:55:18,138 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/info/4b9dfca8b8c84b05950c6e2ead0e83fa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/info/4b9dfca8b8c84b05950c6e2ead0e83fa 2024-11-20T14:55:18,139 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/info/4b9dfca8b8c84b05950c6e2ead0e83fa, delay=954 ms 2024-11-20T14:55:18,140 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/info/4b9dfca8b8c84b05950c6e2ead0e83fa, entries=14, sequenceid=13, filesize=7.5 K 2024-11-20T14:55:18,141 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/ns/cf886fac23e84d78b90935864ca74986 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/ns/cf886fac23e84d78b90935864ca74986 2024-11-20T14:55:18,142 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/ns/cf886fac23e84d78b90935864ca74986, delay=972 ms 2024-11-20T14:55:18,143 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/ns/cf886fac23e84d78b90935864ca74986, entries=2, sequenceid=13, filesize=5.0 K 2024-11-20T14:55:18,143 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/.tmp/table/6c10f51b8c4f4ae697e09316eaacfd7d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/table/6c10f51b8c4f4ae697e09316eaacfd7d 2024-11-20T14:55:18,144 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/table/6c10f51b8c4f4ae697e09316eaacfd7d, delay=983 ms 2024-11-20T14:55:18,145 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/table/6c10f51b8c4f4ae697e09316eaacfd7d, entries=2, sequenceid=13, filesize=5.3 K 2024-11-20T14:55:18,146 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=13, compaction requested=false 2024-11-20T14:55:18,147 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/info/4b9dfca8b8c84b05950c6e2ead0e83fa 2024-11-20T14:55:18,148 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/ns/cf886fac23e84d78b90935864ca74986 2024-11-20T14:55:18,148 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/table/6c10f51b8c4f4ae697e09316eaacfd7d 2024-11-20T14:55:18,152 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-20T14:55:18,152 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T14:55:18,152 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T14:55:18,153 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732114518071Running coprocessor pre-close hooks at 1732114518071Disabling compacts and flushes for region at 1732114518071Disabling writes for close at 1732114518071Obtaining lock to block concurrent updates at 1732114518072 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732114518072Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732114518072Flushing stores of hbase:meta,,1.1588230740 at 1732114518072Flushing 1588230740/info: creating writer at 1732114518072Flushing 1588230740/info: appending metadata at 1732114518092 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732114518093 (+1 ms)Flushing 1588230740/ns: creating writer at 1732114518096 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732114518114 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732114518114Flushing 1588230740/table: creating writer at 1732114518118 (+4 ms)Flushing 1588230740/table: appending metadata at 1732114518135 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732114518135Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@419b251: reopening flushed file at 1732114518138 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bc97ebf: reopening flushed file at 1732114518141 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@93773c2: reopening flushed file at 1732114518143 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=13, compaction requested=false at 1732114518146 (+3 ms)Writing region close event to WAL at 1732114518149 (+3 ms)Running coprocessor post-close hooks at 1732114518152 (+3 ms)Closed at 1732114518152 2024-11-20T14:55:18,153 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T14:55:18,153 INFO [regionserver/1a15ecfd95f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T14:55:18,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1015a0791bd0003, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:18,203 INFO [RS:2;1a15ecfd95f4:39727 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T14:55:18,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39727-0x1015a0791bd0003, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:18,203 INFO [RS:2;1a15ecfd95f4:39727 {}] regionserver.HRegionServer(1031): Exiting; stopping=1a15ecfd95f4,39727,1732114517351; zookeeper connection closed. 2024-11-20T14:55:18,204 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e964e3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e964e3 2024-11-20T14:55:18,271 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(976): stopping server 1a15ecfd95f4,38417,1732114513878; all regions closed. 2024-11-20T14:55:18,273 DEBUG [RS:1;1a15ecfd95f4:38417 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs 2024-11-20T14:55:18,274 INFO [RS:1;1a15ecfd95f4:38417 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a15ecfd95f4%2C38417%2C1732114513878.meta:.meta(num 1732114514968) 2024-11-20T14:55:18,275 DEBUG [RS:1;1a15ecfd95f4:38417 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/oldWALs 2024-11-20T14:55:18,275 INFO [RS:1;1a15ecfd95f4:38417 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a15ecfd95f4%2C38417%2C1732114513878:(num 1732114514696) 2024-11-20T14:55:18,275 DEBUG [RS:1;1a15ecfd95f4:38417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T14:55:18,276 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T14:55:18,276 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T14:55:18,276 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.ChoreService(370): Chore service for: regionserver/1a15ecfd95f4:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T14:55:18,276 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T14:55:18,276 INFO [regionserver/1a15ecfd95f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T14:55:18,276 INFO [RS:1;1a15ecfd95f4:38417 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38417 2024-11-20T14:55:18,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T14:55:18,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a15ecfd95f4,38417,1732114513878 2024-11-20T14:55:18,295 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T14:55:18,307 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a15ecfd95f4,38417,1732114513878] 2024-11-20T14:55:18,315 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1a15ecfd95f4,38417,1732114513878 already deleted, retry=false 2024-11-20T14:55:18,315 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1a15ecfd95f4,38417,1732114513878 expired; onlineServers=0 2024-11-20T14:55:18,315 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1a15ecfd95f4,37933,1732114513673' ***** 2024-11-20T14:55:18,315 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T14:55:18,315 INFO [M:0;1a15ecfd95f4:37933 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T14:55:18,316 INFO [M:0;1a15ecfd95f4:37933 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T14:55:18,316 DEBUG [M:0;1a15ecfd95f4:37933 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T14:55:18,316 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T14:55:18,316 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.small.0-1732114514321 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.small.0-1732114514321,5,FailOnTimeoutGroup] 2024-11-20T14:55:18,316 DEBUG [M:0;1a15ecfd95f4:37933 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T14:55:18,316 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.large.0-1732114514321 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.large.0-1732114514321,5,FailOnTimeoutGroup] 2024-11-20T14:55:18,317 INFO [M:0;1a15ecfd95f4:37933 {}] hbase.ChoreService(370): Chore service for: master/1a15ecfd95f4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T14:55:18,317 INFO [M:0;1a15ecfd95f4:37933 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T14:55:18,317 DEBUG [M:0;1a15ecfd95f4:37933 {}] master.HMaster(1795): Stopping service threads 2024-11-20T14:55:18,318 INFO [M:0;1a15ecfd95f4:37933 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T14:55:18,318 INFO [M:0;1a15ecfd95f4:37933 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T14:55:18,319 INFO [M:0;1a15ecfd95f4:37933 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T14:55:18,319 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T14:55:18,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T14:55:18,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T14:55:18,328 DEBUG [M:0;1a15ecfd95f4:37933 {}] zookeeper.ZKUtil(347): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T14:55:18,328 WARN [M:0;1a15ecfd95f4:37933 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T14:55:18,328 INFO [M:0;1a15ecfd95f4:37933 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/.lastflushedseqids 2024-11-20T14:55:18,333 INFO [M:0;1a15ecfd95f4:37933 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T14:55:18,333 INFO [M:0;1a15ecfd95f4:37933 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T14:55:18,334 DEBUG [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T14:55:18,334 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:18,334 DEBUG [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:18,334 DEBUG [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T14:55:18,334 DEBUG [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:18,334 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-11-20T14:55:18,342 DEBUG [M:0;1a15ecfd95f4:37933 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e2fc97c2ec184894abfcb3f26e8f6b7c 2024-11-20T14:55:18,354 DEBUG [M:0;1a15ecfd95f4:37933 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e2fc97c2ec184894abfcb3f26e8f6b7c is 82, key is hbase:meta,,1/info:regioninfo/1732114515005/Put/seqid=0 2024-11-20T14:55:18,354 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e2fc97c2ec184894abfcb3f26e8f6b7c 2024-11-20T14:55:18,361 DEBUG [M:0;1a15ecfd95f4:37933 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2bffd5137c3847d89bd256c26c1089db 2024-11-20T14:55:18,374 DEBUG [M:0;1a15ecfd95f4:37933 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2bffd5137c3847d89bd256c26c1089db is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732114515534/Put/seqid=0 2024-11-20T14:55:18,374 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2bffd5137c3847d89bd256c26c1089db 2024-11-20T14:55:18,381 DEBUG [M:0;1a15ecfd95f4:37933 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0f86b6081a204fd6bf518f8e4a379388 2024-11-20T14:55:18,400 DEBUG [M:0;1a15ecfd95f4:37933 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0f86b6081a204fd6bf518f8e4a379388 is 69, key is 1a15ecfd95f4,38417,1732114513878/rs:state/1732114514493/Put/seqid=0 2024-11-20T14:55:18,401 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0f86b6081a204fd6bf518f8e4a379388 2024-11-20T14:55:18,403 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f86b6081a204fd6bf518f8e4a379388 2024-11-20T14:55:18,404 DEBUG [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e2fc97c2ec184894abfcb3f26e8f6b7c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e2fc97c2ec184894abfcb3f26e8f6b7c 2024-11-20T14:55:18,406 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e2fc97c2ec184894abfcb3f26e8f6b7c, entries=8, sequenceid=83, filesize=5.5 K 2024-11-20T14:55:18,407 DEBUG [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2bffd5137c3847d89bd256c26c1089db as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2bffd5137c3847d89bd256c26c1089db 2024-11-20T14:55:18,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:18,407 INFO [RS:1;1a15ecfd95f4:38417 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T14:55:18,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38417-0x1015a0791bd0002, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:18,407 INFO [RS:1;1a15ecfd95f4:38417 {}] regionserver.HRegionServer(1031): Exiting; stopping=1a15ecfd95f4,38417,1732114513878; zookeeper connection closed. 2024-11-20T14:55:18,407 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1a99fe67 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1a99fe67 2024-11-20T14:55:18,408 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T14:55:18,408 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2bffd5137c3847d89bd256c26c1089db, entries=9, sequenceid=83, filesize=6.4 K 2024-11-20T14:55:18,408 DEBUG [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0f86b6081a204fd6bf518f8e4a379388 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0f86b6081a204fd6bf518f8e4a379388 2024-11-20T14:55:18,410 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f86b6081a204fd6bf518f8e4a379388 2024-11-20T14:55:18,410 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecf9bd63-fe9e-e876-c5c5-038224d1ef11/40bdbed0-0b40-1552-9815-27ab3a1f10f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0f86b6081a204fd6bf518f8e4a379388, entries=3, sequenceid=83, filesize=5.3 K 2024-11-20T14:55:18,411 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=83, compaction requested=false 2024-11-20T14:55:18,413 INFO [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T14:55:18,413 DEBUG [M:0;1a15ecfd95f4:37933 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732114518333Disabling compacts and flushes for region at 1732114518333Disabling writes for close at 1732114518334 (+1 ms)Obtaining lock to block concurrent updates at 1732114518334Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732114518334Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1732114518334Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732114518335 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732114518335Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732114518353 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732114518353Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732114518356 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732114518373 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732114518373Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732114518376 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732114518399 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732114518399Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27a7fac2: reopening flushed file at 1732114518403 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6da3d1d5: reopening flushed file at 1732114518406 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c4c54ab: reopening flushed file at 1732114518408 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=83, compaction requested=false at 1732114518411 (+3 ms)Writing region close event to WAL at 1732114518412 (+1 ms)Closed at 1732114518412 2024-11-20T14:55:18,414 INFO [M:0;1a15ecfd95f4:37933 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T14:55:18,414 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T14:55:18,414 INFO [M:0;1a15ecfd95f4:37933 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37933 2024-11-20T14:55:18,414 INFO [M:0;1a15ecfd95f4:37933 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T14:55:18,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:18,545 INFO [M:0;1a15ecfd95f4:37933 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T14:55:18,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37933-0x1015a0791bd0000, quorum=127.0.0.1:64993, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T14:55:18,552 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T14:55:18,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T14:55:18,554 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T14:55:18,561 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 381), ProcessCount=11 (was 11), AvailableMemoryMB=10659 (was 10793)