2024-12-06 00:45:37,970 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-06 00:45:37,981 main DEBUG Took 0.009687 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-06 00:45:37,982 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-06 00:45:37,982 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-06 00:45:37,992 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-06 00:45:37,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,003 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-06 00:45:38,019 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,021 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,023 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,024 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,025 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,025 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,026 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,027 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,028 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,029 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,029 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,030 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,031 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,031 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,032 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 00:45:38,033 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,033 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-06 00:45:38,035 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 00:45:38,037 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-06 00:45:38,039 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-06 00:45:38,040 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-06 00:45:38,041 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-06 00:45:38,042 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-06 00:45:38,053 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-06 00:45:38,056 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-06 00:45:38,058 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-06 00:45:38,058 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-06 00:45:38,059 main DEBUG createAppenders(={Console}) 2024-12-06 00:45:38,060 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-06 00:45:38,060 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-06 00:45:38,060 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-06 00:45:38,061 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-06 00:45:38,061 main DEBUG OutputStream closed 2024-12-06 00:45:38,062 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-06 00:45:38,062 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-06 00:45:38,062 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-06 00:45:38,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-06 00:45:38,154 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-06 00:45:38,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-06 00:45:38,157 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-06 00:45:38,157 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-06 00:45:38,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-06 00:45:38,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-06 00:45:38,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-06 00:45:38,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-06 00:45:38,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-06 00:45:38,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-06 00:45:38,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-06 00:45:38,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-06 00:45:38,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-06 00:45:38,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-06 00:45:38,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-06 00:45:38,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-06 00:45:38,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-06 00:45:38,166 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06 00:45:38,166 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-06 00:45:38,167 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-06 00:45:38,168 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-06T00:45:38,182 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-06 00:45:38,185 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-06 00:45:38,185 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06T00:45:38,465 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee 2024-12-06T00:45:38,492 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1321, ProcessCount=11, AvailableMemoryMB=11964 2024-12-06T00:45:38,592 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-06T00:45:38,682 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:38,697 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/cluster_c1a06c0f-0242-d1a6-a239-bc6e04022868, deleteOnExit=true 2024-12-06T00:45:38,803 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/cluster_c1a06c0f-0242-d1a6-a239-bc6e04022868/zookeeper_0, clientPort=50862, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/cluster_c1a06c0f-0242-d1a6-a239-bc6e04022868/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/cluster_c1a06c0f-0242-d1a6-a239-bc6e04022868/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T00:45:38,816 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50862 2024-12-06T00:45:38,817 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:38,903 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 with version=8 2024-12-06T00:45:38,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/hbase-staging 2024-12-06T00:45:39,006 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-06T00:45:39,296 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f7c814cf3c7e:0 server-side Connection retries=45 2024-12-06T00:45:39,307 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:39,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:39,316 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T00:45:39,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:39,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T00:45:39,461 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T00:45:39,564 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T00:45:39,616 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-06T00:45:39,631 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-06T00:45:39,637 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T00:45:39,686 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14395 (auto-detected) 2024-12-06T00:45:39,688 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-06T00:45:39,713 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37921 2024-12-06T00:45:39,738 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37921 connecting to ZooKeeper ensemble=127.0.0.1:50862 2024-12-06T00:45:39,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379210x0, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T00:45:39,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37921-0x10064c9a73c0000 connected 2024-12-06T00:45:39,815 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:39,816 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:39,824 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:39,829 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7, hbase.cluster.distributed=false 2024-12-06T00:45:39,857 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T00:45:39,863 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37921 2024-12-06T00:45:39,863 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37921 2024-12-06T00:45:39,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37921 2024-12-06T00:45:39,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37921 2024-12-06T00:45:39,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37921 2024-12-06T00:45:39,957 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f7c814cf3c7e:0 server-side Connection retries=45 2024-12-06T00:45:39,959 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:39,960 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:39,960 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T00:45:39,960 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:39,960 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T00:45:39,963 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T00:45:39,966 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T00:45:39,967 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39779 2024-12-06T00:45:39,969 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39779 connecting to ZooKeeper ensemble=127.0.0.1:50862 2024-12-06T00:45:39,969 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:39,970 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:39,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:397790x0, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T00:45:39,975 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:397790x0, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:39,975 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39779-0x10064c9a73c0001 connected 2024-12-06T00:45:39,981 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T00:45:39,996 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/bucket.cache 2024-12-06T00:45:40,004 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T00:45:40,006 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T00:45:40,008 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T00:45:40,010 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T00:45:40,010 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T00:45:40,018 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T00:45:40,022 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T00:45:40,022 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/bucket.persistence is complete. 2024-12-06T00:45:40,025 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T00:45:40,028 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T00:45:40,035 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T00:45:40,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39779 2024-12-06T00:45:40,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39779 2024-12-06T00:45:40,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39779 2024-12-06T00:45:40,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39779 2024-12-06T00:45:40,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39779 2024-12-06T00:45:40,069 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f7c814cf3c7e:0 server-side Connection retries=45 2024-12-06T00:45:40,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:40,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:40,069 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T00:45:40,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:40,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T00:45:40,070 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T00:45:40,070 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T00:45:40,072 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39163 2024-12-06T00:45:40,074 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39163 connecting to ZooKeeper ensemble=127.0.0.1:50862 2024-12-06T00:45:40,074 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:40,075 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:40,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391630x0, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T00:45:40,081 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:391630x0, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:40,081 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39163-0x10064c9a73c0002 connected 2024-12-06T00:45:40,082 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T00:45:40,083 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/bucket.cache 2024-12-06T00:45:40,083 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T00:45:40,083 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T00:45:40,083 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T00:45:40,084 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T00:45:40,084 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T00:45:40,085 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T00:45:40,085 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/bucket.persistence is complete. 2024-12-06T00:45:40,085 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T00:45:40,086 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T00:45:40,087 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T00:45:40,089 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T00:45:40,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39163 2024-12-06T00:45:40,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39163 2024-12-06T00:45:40,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39163 2024-12-06T00:45:40,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39163 2024-12-06T00:45:40,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39163 2024-12-06T00:45:40,118 DEBUG [M:0;f7c814cf3c7e:37921 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f7c814cf3c7e:37921 2024-12-06T00:45:40,119 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f7c814cf3c7e,37921,1733445939096 2024-12-06T00:45:40,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:40,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:40,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:40,129 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f7c814cf3c7e,37921,1733445939096 2024-12-06T00:45:40,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T00:45:40,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T00:45:40,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,165 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T00:45:40,167 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f7c814cf3c7e,37921,1733445939096 from backup master directory 2024-12-06T00:45:40,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f7c814cf3c7e,37921,1733445939096 2024-12-06T00:45:40,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:40,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:40,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:40,171 WARN [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T00:45:40,172 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f7c814cf3c7e,37921,1733445939096 2024-12-06T00:45:40,174 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-06T00:45:40,177 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-06T00:45:40,187 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/hbase.id] with ID: 074e04b6-9a4e-4223-98a6-704ccf4dfe2d 2024-12-06T00:45:40,187 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/.tmp/hbase.id 2024-12-06T00:45:40,194 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/hbase.id] 2024-12-06T00:45:40,238 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:40,243 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T00:45:40,264 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-06T00:45:40,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,303 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T00:45:40,307 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T00:45:40,363 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T00:45:40,370 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T00:45:40,437 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store 2024-12-06T00:45:40,465 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-06T00:45:40,470 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:40,473 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T00:45:40,473 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:40,474 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:40,476 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T00:45:40,477 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:40,477 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:40,478 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733445940473Disabling compacts and flushes for region at 1733445940473Disabling writes for close at 1733445940476 (+3 ms)Writing region close event to WAL at 1733445940477 (+1 ms)Closed at 1733445940477 2024-12-06T00:45:40,479 WARN [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/.initializing 2024-12-06T00:45:40,479 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/WALs/f7c814cf3c7e,37921,1733445939096 2024-12-06T00:45:40,495 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T00:45:40,513 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f7c814cf3c7e%2C37921%2C1733445939096, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/WALs/f7c814cf3c7e,37921,1733445939096, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/oldWALs, maxLogs=10 2024-12-06T00:45:40,571 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/WALs/f7c814cf3c7e,37921,1733445939096/f7c814cf3c7e%2C37921%2C1733445939096.1733445940520 2024-12-06T00:45:40,573 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T00:45:40,573 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T00:45:40,574 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:40,579 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,581 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T00:45:40,663 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:40,664 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:40,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,671 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T00:45:40,672 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:40,672 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:40,672 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T00:45:40,678 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:40,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:40,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T00:45:40,683 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:40,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:40,684 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,688 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,689 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,692 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,693 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,696 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T00:45:40,698 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:40,708 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T00:45:40,709 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71222789, jitterRate=0.06130225956439972}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T00:45:40,719 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733445940600Initializing all the Stores at 1733445940601 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445940602 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445940603 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445940603Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445940603Cleaning up temporary data from old regions at 1733445940693 (+90 ms)Region opened successfully at 1733445940718 (+25 ms) 2024-12-06T00:45:40,721 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T00:45:40,767 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1af41e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7c814cf3c7e/172.17.0.2:0 2024-12-06T00:45:40,813 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T00:45:40,832 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T00:45:40,833 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T00:45:40,838 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T00:45:40,838 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-06T00:45:40,847 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-06T00:45:40,848 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T00:45:40,882 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T00:45:40,893 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T00:45:40,895 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T00:45:40,898 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T00:45:40,900 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T00:45:40,902 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T00:45:40,905 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T00:45:40,910 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T00:45:40,912 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T00:45:40,913 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T00:45:40,915 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T00:45:40,936 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T00:45:40,938 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T00:45:40,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:40,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:40,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:40,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,955 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f7c814cf3c7e,37921,1733445939096, sessionid=0x10064c9a73c0000, setting cluster-up flag (Was=false) 2024-12-06T00:45:40,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,977 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T00:45:40,979 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f7c814cf3c7e,37921,1733445939096 2024-12-06T00:45:40,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:40,991 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T00:45:40,993 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f7c814cf3c7e,37921,1733445939096 2024-12-06T00:45:40,999 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T00:45:41,005 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(746): ClusterId : 074e04b6-9a4e-4223-98a6-704ccf4dfe2d 2024-12-06T00:45:41,005 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(746): ClusterId : 074e04b6-9a4e-4223-98a6-704ccf4dfe2d 2024-12-06T00:45:41,008 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T00:45:41,008 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T00:45:41,016 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T00:45:41,016 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T00:45:41,017 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T00:45:41,017 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T00:45:41,025 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T00:45:41,025 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T00:45:41,025 DEBUG [RS:0;f7c814cf3c7e:39779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df172c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7c814cf3c7e/172.17.0.2:0 2024-12-06T00:45:41,025 DEBUG [RS:1;f7c814cf3c7e:39163 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88dece6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7c814cf3c7e/172.17.0.2:0 2024-12-06T00:45:41,045 DEBUG [RS:0;f7c814cf3c7e:39779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f7c814cf3c7e:39779 2024-12-06T00:45:41,047 DEBUG [RS:1;f7c814cf3c7e:39163 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f7c814cf3c7e:39163 2024-12-06T00:45:41,049 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T00:45:41,049 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T00:45:41,049 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T00:45:41,049 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T00:45:41,049 DEBUG [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T00:45:41,049 DEBUG [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T00:45:41,052 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7c814cf3c7e,37921,1733445939096 with port=39779, startcode=1733445939917 2024-12-06T00:45:41,052 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7c814cf3c7e,37921,1733445939096 with port=39163, startcode=1733445940068 2024-12-06T00:45:41,065 DEBUG [RS:1;f7c814cf3c7e:39163 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T00:45:41,065 DEBUG [RS:0;f7c814cf3c7e:39779 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T00:45:41,095 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T00:45:41,107 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T00:45:41,122 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T00:45:41,127 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f7c814cf3c7e,37921,1733445939096 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T00:45:41,137 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f7c814cf3c7e:0, corePoolSize=5, maxPoolSize=5 2024-12-06T00:45:41,138 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f7c814cf3c7e:0, corePoolSize=5, maxPoolSize=5 2024-12-06T00:45:41,138 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f7c814cf3c7e:0, corePoolSize=5, maxPoolSize=5 2024-12-06T00:45:41,138 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f7c814cf3c7e:0, corePoolSize=5, maxPoolSize=5 2024-12-06T00:45:41,138 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f7c814cf3c7e:0, corePoolSize=10, maxPoolSize=10 2024-12-06T00:45:41,139 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,139 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f7c814cf3c7e:0, corePoolSize=2, maxPoolSize=2 2024-12-06T00:45:41,139 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,150 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T00:45:41,150 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T00:45:41,151 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733445971151 2024-12-06T00:45:41,153 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T00:45:41,155 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T00:45:41,158 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:41,158 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44529, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T00:45:41,158 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59271, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T00:45:41,158 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T00:45:41,160 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T00:45:41,160 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T00:45:41,164 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T00:45:41,164 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T00:45:41,167 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37921 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T00:45:41,166 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,169 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T00:45:41,171 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T00:45:41,171 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T00:45:41,174 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T00:45:41,174 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T00:45:41,174 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37921 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T00:45:41,180 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.large.0-1733445941176,5,FailOnTimeoutGroup] 2024-12-06T00:45:41,181 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.small.0-1733445941180,5,FailOnTimeoutGroup] 2024-12-06T00:45:41,181 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,181 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T00:45:41,182 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,183 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,195 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T00:45:41,196 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 2024-12-06T00:45:41,199 DEBUG [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T00:45:41,199 DEBUG [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T00:45:41,199 WARN [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T00:45:41,199 WARN [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T00:45:41,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:41,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T00:45:41,220 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T00:45:41,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:41,220 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:41,221 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T00:45:41,226 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T00:45:41,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:41,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:41,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T00:45:41,232 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T00:45:41,232 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:41,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:41,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T00:45:41,238 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T00:45:41,238 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:41,239 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:41,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T00:45:41,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740 2024-12-06T00:45:41,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740 2024-12-06T00:45:41,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T00:45:41,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T00:45:41,242 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T00:45:41,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T00:45:41,251 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T00:45:41,251 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64665993, jitterRate=-0.03640161454677582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T00:45:41,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733445941213Initializing all the Stores at 1733445941214 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445941214Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445941214Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445941214Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445941214Cleaning up temporary data from old regions at 1733445941241 (+27 ms)Region opened successfully at 1733445941255 (+14 ms) 2024-12-06T00:45:41,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T00:45:41,256 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T00:45:41,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T00:45:41,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T00:45:41,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T00:45:41,258 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T00:45:41,258 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733445941256Disabling compacts and flushes for region at 1733445941256Disabling writes for close at 1733445941256Writing region close event to WAL at 1733445941257 (+1 ms)Closed at 1733445941258 (+1 ms) 2024-12-06T00:45:41,261 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T00:45:41,261 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T00:45:41,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T00:45:41,282 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T00:45:41,284 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T00:45:41,301 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7c814cf3c7e,37921,1733445939096 with port=39163, startcode=1733445940068 2024-12-06T00:45:41,301 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7c814cf3c7e,37921,1733445939096 with port=39779, startcode=1733445939917 2024-12-06T00:45:41,303 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37921 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:41,306 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37921 {}] master.ServerManager(517): Registering regionserver=f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:41,314 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37921 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:41,315 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37921 {}] master.ServerManager(517): Registering regionserver=f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:41,315 DEBUG [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 2024-12-06T00:45:41,315 DEBUG [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T00:45:41,315 DEBUG [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T00:45:41,317 DEBUG [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 2024-12-06T00:45:41,317 DEBUG [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T00:45:41,317 DEBUG [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T00:45:41,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T00:45:41,324 DEBUG [RS:1;f7c814cf3c7e:39163 {}] zookeeper.ZKUtil(111): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:41,324 DEBUG [RS:0;f7c814cf3c7e:39779 {}] zookeeper.ZKUtil(111): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:41,324 WARN [RS:0;f7c814cf3c7e:39779 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T00:45:41,324 WARN [RS:1;f7c814cf3c7e:39163 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T00:45:41,324 INFO [RS:0;f7c814cf3c7e:39779 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T00:45:41,324 INFO [RS:1;f7c814cf3c7e:39163 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T00:45:41,324 DEBUG [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:41,324 DEBUG [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:41,329 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f7c814cf3c7e,39163,1733445940068] 2024-12-06T00:45:41,329 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f7c814cf3c7e,39779,1733445939917] 2024-12-06T00:45:41,356 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T00:45:41,356 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T00:45:41,370 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T00:45:41,370 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T00:45:41,376 INFO [RS:0;f7c814cf3c7e:39779 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T00:45:41,376 INFO [RS:1;f7c814cf3c7e:39163 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T00:45:41,376 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,376 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,377 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T00:45:41,377 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T00:45:41,384 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T00:45:41,384 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T00:45:41,386 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,386 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,386 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,386 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,386 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,386 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,386 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,387 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,387 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,387 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,387 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,387 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,387 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f7c814cf3c7e:0, corePoolSize=2, maxPoolSize=2 2024-12-06T00:45:41,387 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,387 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f7c814cf3c7e:0, corePoolSize=2, maxPoolSize=2 2024-12-06T00:45:41,387 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,388 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,388 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,388 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,388 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,388 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,388 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,388 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,388 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,389 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,389 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f7c814cf3c7e:0, corePoolSize=3, maxPoolSize=3 2024-12-06T00:45:41,389 DEBUG [RS:0;f7c814cf3c7e:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f7c814cf3c7e:0, corePoolSize=3, maxPoolSize=3 2024-12-06T00:45:41,389 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:41,389 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f7c814cf3c7e:0, corePoolSize=3, maxPoolSize=3 2024-12-06T00:45:41,389 DEBUG [RS:1;f7c814cf3c7e:39163 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f7c814cf3c7e:0, corePoolSize=3, maxPoolSize=3 2024-12-06T00:45:41,392 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,392 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,392 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,392 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,392 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,392 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,39779,1733445939917-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T00:45:41,396 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,396 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,396 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,396 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,396 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,396 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,39163,1733445940068-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T00:45:41,417 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T00:45:41,419 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,39163,1733445940068-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,420 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,420 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.Replication(171): f7c814cf3c7e,39163,1733445940068 started 2024-12-06T00:45:41,420 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T00:45:41,421 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,39779,1733445939917-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,421 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,421 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.Replication(171): f7c814cf3c7e,39779,1733445939917 started 2024-12-06T00:45:41,435 WARN [f7c814cf3c7e:37921 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T00:45:41,450 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,450 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:41,450 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(1482): Serving as f7c814cf3c7e,39779,1733445939917, RpcServer on f7c814cf3c7e/172.17.0.2:39779, sessionid=0x10064c9a73c0001 2024-12-06T00:45:41,451 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(1482): Serving as f7c814cf3c7e,39163,1733445940068, RpcServer on f7c814cf3c7e/172.17.0.2:39163, sessionid=0x10064c9a73c0002 2024-12-06T00:45:41,452 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T00:45:41,452 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T00:45:41,452 DEBUG [RS:0;f7c814cf3c7e:39779 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:41,452 DEBUG [RS:1;f7c814cf3c7e:39163 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:41,452 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7c814cf3c7e,39779,1733445939917' 2024-12-06T00:45:41,452 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7c814cf3c7e,39163,1733445940068' 2024-12-06T00:45:41,452 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T00:45:41,452 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T00:45:41,453 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T00:45:41,454 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T00:45:41,454 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T00:45:41,454 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T00:45:41,454 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T00:45:41,454 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T00:45:41,455 DEBUG [RS:1;f7c814cf3c7e:39163 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:41,455 DEBUG [RS:0;f7c814cf3c7e:39779 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:41,455 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7c814cf3c7e,39163,1733445940068' 2024-12-06T00:45:41,455 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T00:45:41,455 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7c814cf3c7e,39779,1733445939917' 2024-12-06T00:45:41,455 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T00:45:41,456 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T00:45:41,456 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T00:45:41,456 DEBUG [RS:1;f7c814cf3c7e:39163 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T00:45:41,457 INFO [RS:1;f7c814cf3c7e:39163 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T00:45:41,457 DEBUG [RS:0;f7c814cf3c7e:39779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T00:45:41,457 INFO [RS:1;f7c814cf3c7e:39163 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T00:45:41,457 INFO [RS:0;f7c814cf3c7e:39779 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T00:45:41,457 INFO [RS:0;f7c814cf3c7e:39779 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T00:45:41,563 INFO [RS:1;f7c814cf3c7e:39163 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T00:45:41,563 INFO [RS:0;f7c814cf3c7e:39779 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T00:45:41,565 INFO [RS:1;f7c814cf3c7e:39163 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7c814cf3c7e%2C39163%2C1733445940068, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39163,1733445940068, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs, maxLogs=55 2024-12-06T00:45:41,565 INFO [RS:0;f7c814cf3c7e:39779 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7c814cf3c7e%2C39779%2C1733445939917, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39779,1733445939917, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs, maxLogs=55 2024-12-06T00:45:41,589 INFO [RS:1;f7c814cf3c7e:39163 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39163,1733445940068/f7c814cf3c7e%2C39163%2C1733445940068.1733445941569 2024-12-06T00:45:41,590 INFO [RS:0;f7c814cf3c7e:39779 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39779,1733445939917/f7c814cf3c7e%2C39779%2C1733445939917.1733445941569 2024-12-06T00:45:41,590 DEBUG [RS:1;f7c814cf3c7e:39163 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T00:45:41,590 DEBUG [RS:0;f7c814cf3c7e:39779 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T00:45:41,687 DEBUG [f7c814cf3c7e:37921 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-06T00:45:41,699 DEBUG [f7c814cf3c7e:37921 {}] balancer.BalancerClusterState(204): Hosts are {f7c814cf3c7e=0} racks are {/default-rack=0} 2024-12-06T00:45:41,703 DEBUG [f7c814cf3c7e:37921 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T00:45:41,703 DEBUG [f7c814cf3c7e:37921 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T00:45:41,703 DEBUG [f7c814cf3c7e:37921 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T00:45:41,703 DEBUG [f7c814cf3c7e:37921 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T00:45:41,703 INFO [f7c814cf3c7e:37921 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T00:45:41,703 INFO [f7c814cf3c7e:37921 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T00:45:41,703 DEBUG [f7c814cf3c7e:37921 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T00:45:41,714 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:41,721 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f7c814cf3c7e,39163,1733445940068, state=OPENING 2024-12-06T00:45:41,728 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T00:45:41,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:41,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:41,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:41,733 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:41,734 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:41,734 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:41,735 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T00:45:41,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f7c814cf3c7e,39163,1733445940068}] 2024-12-06T00:45:41,914 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T00:45:41,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59093, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T00:45:41,926 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T00:45:41,926 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T00:45:41,927 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T00:45:41,929 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7c814cf3c7e%2C39163%2C1733445940068.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39163,1733445940068, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs, maxLogs=55 2024-12-06T00:45:41,955 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39163,1733445940068/f7c814cf3c7e%2C39163%2C1733445940068.meta.1733445941930.meta 2024-12-06T00:45:41,955 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T00:45:41,956 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T00:45:41,957 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T00:45:41,961 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T00:45:41,966 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T00:45:41,970 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T00:45:41,971 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:41,971 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T00:45:41,971 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T00:45:41,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T00:45:41,974 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T00:45:41,974 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:41,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:41,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T00:45:41,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T00:45:41,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:41,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:41,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T00:45:41,977 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T00:45:41,977 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:41,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:41,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T00:45:41,978 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T00:45:41,978 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:41,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:41,978 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T00:45:41,980 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740 2024-12-06T00:45:41,981 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740 2024-12-06T00:45:41,981 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T00:45:41,981 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T00:45:41,982 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T00:45:41,983 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T00:45:41,984 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65595524, jitterRate=-0.022550523281097412}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T00:45:41,984 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T00:45:41,986 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733445941972Writing region info on filesystem at 1733445941972Initializing all the Stores at 1733445941972Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445941972Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445941973 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445941973Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445941973Cleaning up temporary data from old regions at 1733445941982 (+9 ms)Running coprocessor post-open hooks at 1733445941984 (+2 ms)Region opened successfully at 1733445941986 (+2 ms) 2024-12-06T00:45:41,992 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733445941905 2024-12-06T00:45:42,003 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T00:45:42,004 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T00:45:42,005 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:42,007 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f7c814cf3c7e,39163,1733445940068, state=OPEN 2024-12-06T00:45:42,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T00:45:42,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T00:45:42,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T00:45:42,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:42,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:42,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:42,010 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:42,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T00:45:42,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f7c814cf3c7e,39163,1733445940068 in 272 msec 2024-12-06T00:45:42,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T00:45:42,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 745 msec 2024-12-06T00:45:42,019 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T00:45:42,019 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T00:45:42,042 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T00:45:42,043 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f7c814cf3c7e,39163,1733445940068, seqNum=-1] 2024-12-06T00:45:42,064 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T00:45:42,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41891, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T00:45:42,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0480 sec 2024-12-06T00:45:42,083 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733445942083, completionTime=-1 2024-12-06T00:45:42,087 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T00:45:42,087 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T00:45:42,115 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T00:45:42,115 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733446002115 2024-12-06T00:45:42,115 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733446062115 2024-12-06T00:45:42,115 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-06T00:45:42,118 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-06T00:45:42,125 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,37921,1733445939096-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:42,126 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,37921,1733445939096-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:42,126 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,37921,1733445939096-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:42,127 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f7c814cf3c7e:37921, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:42,128 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:42,128 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:42,134 DEBUG [master/f7c814cf3c7e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T00:45:42,156 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.984sec 2024-12-06T00:45:42,158 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T00:45:42,159 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T00:45:42,160 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T00:45:42,161 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T00:45:42,161 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T00:45:42,162 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,37921,1733445939096-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T00:45:42,162 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,37921,1733445939096-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T00:45:42,164 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T00:45:42,164 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T00:45:42,165 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,37921,1733445939096-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:42,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T00:45:42,240 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-06T00:45:42,240 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-06T00:45:42,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f7c814cf3c7e,37921,-1 for getting cluster id 2024-12-06T00:45:42,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T00:45:42,256 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '074e04b6-9a4e-4223-98a6-704ccf4dfe2d' 2024-12-06T00:45:42,259 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T00:45:42,259 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "074e04b6-9a4e-4223-98a6-704ccf4dfe2d" 2024-12-06T00:45:42,261 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@571d2278, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T00:45:42,261 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f7c814cf3c7e,37921,-1] 2024-12-06T00:45:42,264 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T00:45:42,265 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:42,268 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T00:45:42,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T00:45:42,271 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T00:45:42,278 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f7c814cf3c7e,39163,1733445940068, seqNum=-1] 2024-12-06T00:45:42,279 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T00:45:42,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T00:45:42,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f7c814cf3c7e,37921,1733445939096 2024-12-06T00:45:42,311 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T00:45:42,316 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f7c814cf3c7e,37921,1733445939096 2024-12-06T00:45:42,319 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3aeefebb 2024-12-06T00:45:42,320 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T00:45:42,323 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T00:45:42,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733445942306', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T00:45:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733445942306 2024-12-06T00:45:42,337 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733445942306 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T00:45:42,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733445942306" procId is: 4 2024-12-06T00:45:42,339 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:42,341 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733445942306 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T00:45:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T00:45:42,355 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733445942306-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9d9fcc432901580cbab21b92352b01d8, NAME => 'testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733445942306', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 2024-12-06T00:45:42,362 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733445942306-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:42,363 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733445942306-pool-0 {}] regionserver.HRegion(1722): Closing 9d9fcc432901580cbab21b92352b01d8, disabling compactions & flushes 2024-12-06T00:45:42,363 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733445942306-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:42,363 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733445942306-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:42,363 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733445942306-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. after waiting 0 ms 2024-12-06T00:45:42,363 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733445942306-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:42,363 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733445942306-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:42,363 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733445942306-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9d9fcc432901580cbab21b92352b01d8: Waiting for close lock at 1733445942362Disabling compacts and flushes for region at 1733445942362Disabling writes for close at 1733445942363 (+1 ms)Writing region close event to WAL at 1733445942363Closed at 1733445942363 2024-12-06T00:45:42,365 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733445942306 execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T00:45:42,370 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733445942365"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733445942365"}]},"ts":"1733445942365"} 2024-12-06T00:45:42,375 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T00:45:42,376 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733445942306 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T00:45:42,379 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733445942306","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733445942376"}]},"ts":"1733445942376"} 2024-12-06T00:45:42,383 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733445942306, state=ENABLING in hbase:meta 2024-12-06T00:45:42,384 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f7c814cf3c7e=0} racks are {/default-rack=0} 2024-12-06T00:45:42,384 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T00:45:42,384 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T00:45:42,384 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T00:45:42,384 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T00:45:42,384 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T00:45:42,384 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T00:45:42,384 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T00:45:42,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, ASSIGN}] 2024-12-06T00:45:42,387 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, ASSIGN 2024-12-06T00:45:42,389 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, ASSIGN; state=OFFLINE, location=f7c814cf3c7e,39779,1733445939917; forceNewPlan=false, retain=false 2024-12-06T00:45:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T00:45:42,541 INFO [f7c814cf3c7e:37921 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T00:45:42,542 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9d9fcc432901580cbab21b92352b01d8, regionState=OPENING, regionLocation=f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:42,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, ASSIGN because future has completed 2024-12-06T00:45:42,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d9fcc432901580cbab21b92352b01d8, server=f7c814cf3c7e,39779,1733445939917}] 2024-12-06T00:45:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T00:45:42,701 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T00:45:42,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58185, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T00:45:42,706 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:42,707 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d9fcc432901580cbab21b92352b01d8, NAME => 'testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8.', STARTKEY => '', ENDKEY => ''} 2024-12-06T00:45:42,708 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733445942306 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,708 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:42,708 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,708 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,711 INFO [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,715 INFO [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d9fcc432901580cbab21b92352b01d8 columnFamilyName family 2024-12-06T00:45:42,716 DEBUG [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:42,716 INFO [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] regionserver.HStore(327): Store=9d9fcc432901580cbab21b92352b01d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:42,717 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,718 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,718 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,718 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,718 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,719 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,726 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T00:45:42,726 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9d9fcc432901580cbab21b92352b01d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65449775, jitterRate=-0.024722352623939514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T00:45:42,727 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:42,728 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9d9fcc432901580cbab21b92352b01d8: Running coprocessor pre-open hook at 1733445942708Writing region info on filesystem at 1733445942708Initializing all the Stores at 1733445942709 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445942709Cleaning up temporary data from old regions at 1733445942718 (+9 ms)Running coprocessor post-open hooks at 1733445942727 (+9 ms)Region opened successfully at 1733445942728 (+1 ms) 2024-12-06T00:45:42,729 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8., pid=6, masterSystemTime=1733445942701 2024-12-06T00:45:42,733 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:42,733 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:42,733 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9d9fcc432901580cbab21b92352b01d8, regionState=OPEN, openSeqNum=2, regionLocation=f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:42,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d9fcc432901580cbab21b92352b01d8, server=f7c814cf3c7e,39779,1733445939917 because future has completed 2024-12-06T00:45:42,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T00:45:42,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d9fcc432901580cbab21b92352b01d8, server=f7c814cf3c7e,39779,1733445939917 in 192 msec 2024-12-06T00:45:42,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T00:45:42,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, ASSIGN in 355 msec 2024-12-06T00:45:42,745 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733445942306 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T00:45:42,745 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733445942306","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733445942745"}]},"ts":"1733445942745"} 2024-12-06T00:45:42,748 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733445942306, state=ENABLED in hbase:meta 2024-12-06T00:45:42,749 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733445942306 execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T00:45:42,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733445942306 in 419 msec 2024-12-06T00:45:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T00:45:42,978 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733445942306 completed 2024-12-06T00:45:42,978 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733445942306 get assigned. Timeout = 60000ms 2024-12-06T00:45:42,981 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T00:45:42,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733445942306 assigned to meta. Checking AM states. 2024-12-06T00:45:42,988 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T00:45:42,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733445942306 assigned. 2024-12-06T00:45:42,998 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733445942306', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8., hostname=f7c814cf3c7e,39779,1733445939917, seqNum=2] 2024-12-06T00:45:43,000 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T00:45:43,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T00:45:43,013 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9d9fcc432901580cbab21b92352b01d8 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-06T00:45:43,042 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5d8e7927099243d0955cd75b54680d63 2024-12-06T00:45:43,082 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/.tmp/family/5d8e7927099243d0955cd75b54680d63 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-06T00:45:43,087 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/.tmp/family/5d8e7927099243d0955cd75b54680d63 2024-12-06T00:45:43,124 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/.tmp/family/5d8e7927099243d0955cd75b54680d63 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63 2024-12-06T00:45:43,126 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63, delay=913 ms 2024-12-06T00:45:43,128 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63, entries=2, sequenceid=6, filesize=4.9 K 2024-12-06T00:45:43,133 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 9d9fcc432901580cbab21b92352b01d8 in 121ms, sequenceid=6, compaction requested=false 2024-12-06T00:45:43,134 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733445942306' 2024-12-06T00:45:43,136 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9d9fcc432901580cbab21b92352b01d8: 2024-12-06T00:45:44,041 DEBUG [hfile-prefetch-1733445943126 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5d8e7927099243d0955cd75b54680d63_0. 2024-12-06T00:45:44,046 DEBUG [hfile-prefetch-1733445943126 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5d8e7927099243d0955cd75b54680d63_117. 2024-12-06T00:45:44,047 DEBUG [hfile-prefetch-1733445943126 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63, with total blocks 2, and data blocks 1 2024-12-06T00:45:44,047 DEBUG [hfile-prefetch-1733445943126 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-06T00:45:44,047 DEBUG [hfile-prefetch-1733445943126 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-06T00:45:44,048 DEBUG [hfile-prefetch-1733445943126 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63. Will sleep 100ms and try the verification again. 2024-12-06T00:45:44,087 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-06T00:45:44,087 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-06T00:45:44,087 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T00:45:44,087 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T00:45:44,106 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T00:45:44,106 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733445944087 2024-12-06T00:45:44,106 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-06T00:45:44,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=9d9fcc432901580cbab21b92352b01d8, source=f7c814cf3c7e,39779,1733445939917, destination=f7c814cf3c7e,39163,1733445940068, warming up region on f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:44,148 DEBUG [hfile-prefetch-1733445943126 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63, with total blocks 2, and data blocks 1 2024-12-06T00:45:44,148 DEBUG [hfile-prefetch-1733445943126 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-06T00:45:44,148 DEBUG [hfile-prefetch-1733445943126 {}] bucket.BucketCache(2293): found block for file 5d8e7927099243d0955cd75b54680d63 in the backing map. Acquiring read lock for offset 0 2024-12-06T00:45:44,149 DEBUG [hfile-prefetch-1733445943126 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63 has now been fully cached. 2024-12-06T00:45:44,149 DEBUG [hfile-prefetch-1733445943126 {}] hfile.PrefetchExecutor(121): Prefetch completed for 5d8e7927099243d0955cd75b54680d63 2024-12-06T00:45:44,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=9d9fcc432901580cbab21b92352b01d8, source=f7c814cf3c7e,39779,1733445939917, destination=f7c814cf3c7e,39163,1733445940068, running balancer 2024-12-06T00:45:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, REOPEN/MOVE 2024-12-06T00:45:44,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,151 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, REOPEN/MOVE 2024-12-06T00:45:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] regionserver.HRegion(7855): Warmup {ENCODED => 9d9fcc432901580cbab21b92352b01d8, NAME => 'testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8.', STARTKEY => '', ENDKEY => ''} 2024-12-06T00:45:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:44,153 INFO [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,153 INFO [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d9fcc432901580cbab21b92352b01d8 columnFamilyName family 2024-12-06T00:45:44,154 DEBUG [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:44,154 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9d9fcc432901580cbab21b92352b01d8, regionState=CLOSING, regionLocation=f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:44,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, REOPEN/MOVE because future has completed 2024-12-06T00:45:44,157 DEBUG [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63 2024-12-06T00:45:44,158 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-06T00:45:44,158 INFO [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] regionserver.HStore(327): Store=9d9fcc432901580cbab21b92352b01d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:44,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d9fcc432901580cbab21b92352b01d8, server=f7c814cf3c7e,39779,1733445939917}] 2024-12-06T00:45:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] regionserver.HRegion(1722): Closing 9d9fcc432901580cbab21b92352b01d8, disabling compactions & flushes 2024-12-06T00:45:44,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. after waiting 0 ms 2024-12-06T00:45:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39163 {}] regionserver.HRegion(1676): Region close journal for 9d9fcc432901580cbab21b92352b01d8: Waiting for close lock at 1733445944158Disabling compacts and flushes for region at 1733445944158Disabling writes for close at 1733445944158Writing region close event to WAL at 1733445944160 (+2 ms)Closed at 1733445944160 2024-12-06T00:45:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37921 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-06T00:45:44,318 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,318 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-06T00:45:44,319 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 9d9fcc432901580cbab21b92352b01d8, disabling compactions & flushes 2024-12-06T00:45:44,319 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,319 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,319 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. after waiting 0 ms 2024-12-06T00:45:44,319 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,321 DEBUG [StoreFileCloser-9d9fcc432901580cbab21b92352b01d8-family-1 {}] bucket.BucketCache(768): Removing file 5d8e7927099243d0955cd75b54680d63 for region 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,321 DEBUG [StoreFileCloser-9d9fcc432901580cbab21b92352b01d8-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 5d8e7927099243d0955cd75b54680d63, starting offset: 0, end offset: 9223372036854775807 2024-12-06T00:45:44,322 DEBUG [StoreFileCloser-9d9fcc432901580cbab21b92352b01d8-family-1 {}] bucket.BucketCache(856): removed key 5d8e7927099243d0955cd75b54680d63_0 from back map with offset lock 2094080 in the evict process 2024-12-06T00:45:44,324 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-06T00:45:44,324 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-06T00:45:44,324 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T00:45:44,324 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T00:45:44,327 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T00:45:44,327 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733445944324 2024-12-06T00:45:44,327 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-06T00:45:44,328 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-06T00:45:44,329 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,329 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 9d9fcc432901580cbab21b92352b01d8: Waiting for close lock at 1733445944319Running coprocessor pre-close hooks at 1733445944319Disabling compacts and flushes for region at 1733445944319Disabling writes for close at 1733445944319Writing region close event to WAL at 1733445944323 (+4 ms)Running coprocessor post-close hooks at 1733445944328 (+5 ms)Closed at 1733445944329 (+1 ms) 2024-12-06T00:45:44,330 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 9d9fcc432901580cbab21b92352b01d8 move to f7c814cf3c7e,39163,1733445940068 record at close sequenceid=6 2024-12-06T00:45:44,333 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,333 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9d9fcc432901580cbab21b92352b01d8, regionState=CLOSED 2024-12-06T00:45:44,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d9fcc432901580cbab21b92352b01d8, server=f7c814cf3c7e,39779,1733445939917 because future has completed 2024-12-06T00:45:44,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T00:45:44,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 9d9fcc432901580cbab21b92352b01d8, server=f7c814cf3c7e,39779,1733445939917 in 179 msec 2024-12-06T00:45:44,340 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, REOPEN/MOVE; state=CLOSED, location=f7c814cf3c7e,39163,1733445940068; forceNewPlan=false, retain=false 2024-12-06T00:45:44,490 INFO [f7c814cf3c7e:37921 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T00:45:44,491 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9d9fcc432901580cbab21b92352b01d8, regionState=OPENING, regionLocation=f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:44,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, REOPEN/MOVE because future has completed 2024-12-06T00:45:44,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d9fcc432901580cbab21b92352b01d8, server=f7c814cf3c7e,39163,1733445940068}] 2024-12-06T00:45:44,650 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,651 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d9fcc432901580cbab21b92352b01d8, NAME => 'testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8.', STARTKEY => '', ENDKEY => ''} 2024-12-06T00:45:44,651 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733445942306 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,651 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:44,651 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,651 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,652 INFO [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,654 INFO [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d9fcc432901580cbab21b92352b01d8 columnFamilyName family 2024-12-06T00:45:44,654 DEBUG [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:44,656 DEBUG [StoreFileOpener-9d9fcc432901580cbab21b92352b01d8-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63, delay=976 ms 2024-12-06T00:45:44,658 DEBUG [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63 2024-12-06T00:45:44,658 INFO [StoreOpener-9d9fcc432901580cbab21b92352b01d8-1 {}] regionserver.HStore(327): Store=9d9fcc432901580cbab21b92352b01d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:44,659 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,660 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,660 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,661 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,661 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,666 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,667 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 9d9fcc432901580cbab21b92352b01d8; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67844296, jitterRate=0.01095879077911377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T00:45:44,667 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,668 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 9d9fcc432901580cbab21b92352b01d8: Running coprocessor pre-open hook at 1733445944652Writing region info on filesystem at 1733445944652Initializing all the Stores at 1733445944652Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445944652Cleaning up temporary data from old regions at 1733445944661 (+9 ms)Running coprocessor post-open hooks at 1733445944667 (+6 ms)Region opened successfully at 1733445944668 (+1 ms) 2024-12-06T00:45:44,669 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8., pid=9, masterSystemTime=1733445944647 2024-12-06T00:45:44,672 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,672 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,673 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9d9fcc432901580cbab21b92352b01d8, regionState=OPEN, openSeqNum=10, regionLocation=f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:44,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d9fcc432901580cbab21b92352b01d8, server=f7c814cf3c7e,39163,1733445940068 because future has completed 2024-12-06T00:45:44,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-06T00:45:44,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d9fcc432901580cbab21b92352b01d8, server=f7c814cf3c7e,39163,1733445940068 in 183 msec 2024-12-06T00:45:44,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733445942306, region=9d9fcc432901580cbab21b92352b01d8, REOPEN/MOVE in 530 msec 2024-12-06T00:45:44,704 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T00:45:44,705 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T00:45:44,705 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T00:45:44,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:44,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:44,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T00:45:44,711 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T00:45:44,711 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-12-06T00:45:44,711 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f7c814cf3c7e,37921,1733445939096 2024-12-06T00:45:44,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:44,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:44,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:44,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:44,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:44,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:44,714 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T00:45:44,715 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:44,715 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:44,715 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T00:45:44,715 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T00:45:44,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:44,716 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:44,716 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f7c814cf3c7e,39779,1733445939917' ***** 2024-12-06T00:45:44,716 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T00:45:44,716 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f7c814cf3c7e,39163,1733445940068' ***** 2024-12-06T00:45:44,716 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T00:45:44,716 INFO [RS:0;f7c814cf3c7e:39779 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T00:45:44,716 INFO [RS:1;f7c814cf3c7e:39163 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T00:45:44,717 INFO [RS:0;f7c814cf3c7e:39779 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/bucket.persistence 2024-12-06T00:45:44,717 INFO [RS:1;f7c814cf3c7e:39163 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/bucket.persistence 2024-12-06T00:45:44,717 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T00:45:44,717 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T00:45:44,717 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T00:45:44,717 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T00:45:44,717 INFO [RS:0;f7c814cf3c7e:39779 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T00:45:44,717 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T00:45:44,717 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T00:45:44,717 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T00:45:44,718 INFO [RS:1;f7c814cf3c7e:39163 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T00:45:44,718 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T00:45:44,727 DEBUG [RS:0;f7c814cf3c7e:39779 {}] bucket.BucketCache(1487): Thread RS:0;f7c814cf3c7e:39779 started persisting bucket cache to file 2024-12-06T00:45:44,727 DEBUG [RS:0;f7c814cf3c7e:39779 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T00:45:44,727 DEBUG [RS:0;f7c814cf3c7e:39779 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T00:45:44,728 DEBUG [RS:1;f7c814cf3c7e:39163 {}] bucket.BucketCache(1487): Thread RS:1;f7c814cf3c7e:39163 started persisting bucket cache to file 2024-12-06T00:45:44,728 DEBUG [RS:1;f7c814cf3c7e:39163 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T00:45:44,728 DEBUG [RS:1;f7c814cf3c7e:39163 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T00:45:44,731 DEBUG [RS:0;f7c814cf3c7e:39779 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T00:45:44,731 DEBUG [RS:0;f7c814cf3c7e:39779 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733445944727 2024-12-06T00:45:44,731 DEBUG [RS:0;f7c814cf3c7e:39779 {}] bucket.BucketCache(1509): Thread RS:0;f7c814cf3c7e:39779 finished persisting bucket cache to file, renaming 2024-12-06T00:45:44,732 DEBUG [RS:1;f7c814cf3c7e:39163 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T00:45:44,732 DEBUG [RS:1;f7c814cf3c7e:39163 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733445944728 2024-12-06T00:45:44,732 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T00:45:44,732 DEBUG [RS:1;f7c814cf3c7e:39163 {}] bucket.BucketCache(1509): Thread RS:1;f7c814cf3c7e:39163 finished persisting bucket cache to file, renaming 2024-12-06T00:45:44,732 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T00:45:44,732 INFO [RS:0;f7c814cf3c7e:39779 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T00:45:44,732 INFO [RS:0;f7c814cf3c7e:39779 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T00:45:44,732 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(959): stopping server f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:44,732 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T00:45:44,732 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T00:45:44,732 INFO [RS:0;f7c814cf3c7e:39779 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f7c814cf3c7e:39779. 2024-12-06T00:45:44,733 INFO [RS:1;f7c814cf3c7e:39163 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T00:45:44,733 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T00:45:44,733 INFO [RS:1;f7c814cf3c7e:39163 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T00:45:44,733 DEBUG [RS:0;f7c814cf3c7e:39779 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T00:45:44,733 DEBUG [RS:0;f7c814cf3c7e:39779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:44,733 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(3091): Received CLOSE for 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,733 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(976): stopping server f7c814cf3c7e,39779,1733445939917; all regions closed. 2024-12-06T00:45:44,733 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(959): stopping server f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:44,733 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T00:45:44,733 INFO [RS:1;f7c814cf3c7e:39163 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f7c814cf3c7e:39163. 2024-12-06T00:45:44,734 DEBUG [RS:1;f7c814cf3c7e:39163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T00:45:44,734 DEBUG [RS:1;f7c814cf3c7e:39163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:44,734 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T00:45:44,734 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T00:45:44,734 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T00:45:44,734 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T00:45:44,734 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T00:45:44,734 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9d9fcc432901580cbab21b92352b01d8, disabling compactions & flushes 2024-12-06T00:45:44,734 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,734 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,734 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. after waiting 0 ms 2024-12-06T00:45:44,734 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-06T00:45:44,735 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,735 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T00:45:44,735 DEBUG [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 9d9fcc432901580cbab21b92352b01d8=testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8.} 2024-12-06T00:45:44,735 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T00:45:44,735 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T00:45:44,735 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T00:45:44,735 DEBUG [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 9d9fcc432901580cbab21b92352b01d8 2024-12-06T00:45:44,735 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T00:45:44,735 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T00:45:44,735 DEBUG [StoreFileCloser-9d9fcc432901580cbab21b92352b01d8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/family/5d8e7927099243d0955cd75b54680d63 2024-12-06T00:45:44,735 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-06T00:45:44,739 DEBUG [RS:0;f7c814cf3c7e:39779 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs 2024-12-06T00:45:44,739 INFO [RS:0;f7c814cf3c7e:39779 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7c814cf3c7e%2C39779%2C1733445939917:(num 1733445941569) 2024-12-06T00:45:44,739 DEBUG [RS:0;f7c814cf3c7e:39779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:44,739 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T00:45:44,739 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T00:45:44,739 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.ChoreService(370): Chore service for: regionserver/f7c814cf3c7e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-06T00:45:44,739 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T00:45:44,739 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T00:45:44,740 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T00:45:44,740 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T00:45:44,740 INFO [RS:0;f7c814cf3c7e:39779 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39779 2024-12-06T00:45:44,742 INFO [regionserver/f7c814cf3c7e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T00:45:44,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f7c814cf3c7e,39779,1733445939917 2024-12-06T00:45:44,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T00:45:44,745 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnRegionMove1733445942306/9d9fcc432901580cbab21b92352b01d8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-06T00:45:44,745 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T00:45:44,745 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,745 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9d9fcc432901580cbab21b92352b01d8: Waiting for close lock at 1733445944734Running coprocessor pre-close hooks at 1733445944734Disabling compacts and flushes for region at 1733445944734Disabling writes for close at 1733445944735 (+1 ms)Writing region close event to WAL at 1733445944736 (+1 ms)Running coprocessor post-close hooks at 1733445944745 (+9 ms)Closed at 1733445944745 2024-12-06T00:45:44,745 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8. 2024-12-06T00:45:44,748 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f7c814cf3c7e,39779,1733445939917] 2024-12-06T00:45:44,749 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f7c814cf3c7e,39779,1733445939917 already deleted, retry=false 2024-12-06T00:45:44,750 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f7c814cf3c7e,39779,1733445939917 expired; onlineServers=1 2024-12-06T00:45:44,751 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9da446c05f434acf9c0fd552970c3c8c 2024-12-06T00:45:44,768 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/info/9da446c05f434acf9c0fd552970c3c8c is 205, key is testBlockEvictionOnRegionMove1733445942306,,1733445942324.9d9fcc432901580cbab21b92352b01d8./info:regioninfo/1733445944673/Put/seqid=0 2024-12-06T00:45:44,769 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/info/9da446c05f434acf9c0fd552970c3c8c 2024-12-06T00:45:44,779 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2dc856a3c87949adb059fba59fafadd1 2024-12-06T00:45:44,793 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/ns/2dc856a3c87949adb059fba59fafadd1 is 43, key is default/ns:d/1733445942070/Put/seqid=0 2024-12-06T00:45:44,793 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/ns/2dc856a3c87949adb059fba59fafadd1 2024-12-06T00:45:44,798 INFO [regionserver/f7c814cf3c7e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T00:45:44,799 INFO [regionserver/f7c814cf3c7e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T00:45:44,803 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5d4001d383e34208b026063a7840adf1 2024-12-06T00:45:44,816 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/table/5d4001d383e34208b026063a7840adf1 is 78, key is testBlockEvictionOnRegionMove1733445942306/table:state/1733445942745/Put/seqid=0 2024-12-06T00:45:44,817 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/table/5d4001d383e34208b026063a7840adf1 2024-12-06T00:45:44,820 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/info/9da446c05f434acf9c0fd552970c3c8c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/info/9da446c05f434acf9c0fd552970c3c8c 2024-12-06T00:45:44,821 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/info/9da446c05f434acf9c0fd552970c3c8c, delay=901 ms 2024-12-06T00:45:44,823 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/info/9da446c05f434acf9c0fd552970c3c8c, entries=15, sequenceid=15, filesize=7.6 K 2024-12-06T00:45:44,823 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/ns/2dc856a3c87949adb059fba59fafadd1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/ns/2dc856a3c87949adb059fba59fafadd1 2024-12-06T00:45:44,825 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/ns/2dc856a3c87949adb059fba59fafadd1, delay=926 ms 2024-12-06T00:45:44,827 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/ns/2dc856a3c87949adb059fba59fafadd1, entries=2, sequenceid=15, filesize=5.0 K 2024-12-06T00:45:44,828 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/table/5d4001d383e34208b026063a7840adf1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/table/5d4001d383e34208b026063a7840adf1 2024-12-06T00:45:44,829 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/table/5d4001d383e34208b026063a7840adf1, delay=920 ms 2024-12-06T00:45:44,831 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/table/5d4001d383e34208b026063a7840adf1, entries=2, sequenceid=15, filesize=5.3 K 2024-12-06T00:45:44,832 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 96ms, sequenceid=15, compaction requested=false 2024-12-06T00:45:44,832 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-06T00:45:44,834 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/info/9da446c05f434acf9c0fd552970c3c8c 2024-12-06T00:45:44,835 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/ns/2dc856a3c87949adb059fba59fafadd1 2024-12-06T00:45:44,835 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/table/5d4001d383e34208b026063a7840adf1 2024-12-06T00:45:44,840 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-06T00:45:44,840 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T00:45:44,840 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T00:45:44,841 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733445944735Running coprocessor pre-close hooks at 1733445944735Disabling compacts and flushes for region at 1733445944735Disabling writes for close at 1733445944735Obtaining lock to block concurrent updates at 1733445944736 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733445944736Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733445944736Flushing stores of hbase:meta,,1.1588230740 at 1733445944736Flushing 1588230740/info: creating writer at 1733445944737 (+1 ms)Flushing 1588230740/info: appending metadata at 1733445944767 (+30 ms)Flushing 1588230740/info: closing flushed file at 1733445944767Flushing 1588230740/ns: creating writer at 1733445944772 (+5 ms)Flushing 1588230740/ns: appending metadata at 1733445944791 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1733445944791Flushing 1588230740/table: creating writer at 1733445944797 (+6 ms)Flushing 1588230740/table: appending metadata at 1733445944816 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733445944816Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a9c48d6: reopening flushed file at 1733445944820 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cfb544a: reopening flushed file at 1733445944823 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43c85ef0: reopening flushed file at 1733445944827 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 96ms, sequenceid=15, compaction requested=false at 1733445944832 (+5 ms)Writing region close event to WAL at 1733445944835 (+3 ms)Running coprocessor post-close hooks at 1733445944840 (+5 ms)Closed at 1733445944840 2024-12-06T00:45:44,841 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T00:45:44,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:44,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10064c9a73c0001, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:44,849 INFO [RS:0;f7c814cf3c7e:39779 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T00:45:44,850 INFO [RS:0;f7c814cf3c7e:39779 {}] regionserver.HRegionServer(1031): Exiting; stopping=f7c814cf3c7e,39779,1733445939917; zookeeper connection closed. 2024-12-06T00:45:44,850 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@321967ae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@321967ae 2024-12-06T00:45:44,935 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(976): stopping server f7c814cf3c7e,39163,1733445940068; all regions closed. 2024-12-06T00:45:44,938 DEBUG [RS:1;f7c814cf3c7e:39163 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs 2024-12-06T00:45:44,938 INFO [RS:1;f7c814cf3c7e:39163 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7c814cf3c7e%2C39163%2C1733445940068.meta:.meta(num 1733445941930) 2024-12-06T00:45:44,940 DEBUG [RS:1;f7c814cf3c7e:39163 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs 2024-12-06T00:45:44,940 INFO [RS:1;f7c814cf3c7e:39163 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7c814cf3c7e%2C39163%2C1733445940068:(num 1733445941569) 2024-12-06T00:45:44,940 DEBUG [RS:1;f7c814cf3c7e:39163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:44,940 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T00:45:44,940 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T00:45:44,940 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.ChoreService(370): Chore service for: regionserver/f7c814cf3c7e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T00:45:44,941 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T00:45:44,941 INFO [regionserver/f7c814cf3c7e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T00:45:44,941 INFO [RS:1;f7c814cf3c7e:39163 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39163 2024-12-06T00:45:44,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T00:45:44,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f7c814cf3c7e,39163,1733445940068 2024-12-06T00:45:44,944 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T00:45:44,945 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f7c814cf3c7e,39163,1733445940068] 2024-12-06T00:45:44,947 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f7c814cf3c7e,39163,1733445940068 already deleted, retry=false 2024-12-06T00:45:44,947 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f7c814cf3c7e,39163,1733445940068 expired; onlineServers=0 2024-12-06T00:45:44,947 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f7c814cf3c7e,37921,1733445939096' ***** 2024-12-06T00:45:44,947 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T00:45:44,947 INFO [M:0;f7c814cf3c7e:37921 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T00:45:44,948 INFO [M:0;f7c814cf3c7e:37921 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T00:45:44,948 DEBUG [M:0;f7c814cf3c7e:37921 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T00:45:44,948 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T00:45:44,948 DEBUG [M:0;f7c814cf3c7e:37921 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T00:45:44,948 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.large.0-1733445941176 {}] cleaner.HFileCleaner(306): Exit Thread[master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.large.0-1733445941176,5,FailOnTimeoutGroup] 2024-12-06T00:45:44,948 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.small.0-1733445941180 {}] cleaner.HFileCleaner(306): Exit Thread[master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.small.0-1733445941180,5,FailOnTimeoutGroup] 2024-12-06T00:45:44,949 INFO [M:0;f7c814cf3c7e:37921 {}] hbase.ChoreService(370): Chore service for: master/f7c814cf3c7e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T00:45:44,949 INFO [M:0;f7c814cf3c7e:37921 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T00:45:44,949 DEBUG [M:0;f7c814cf3c7e:37921 {}] master.HMaster(1795): Stopping service threads 2024-12-06T00:45:44,949 INFO [M:0;f7c814cf3c7e:37921 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T00:45:44,949 INFO [M:0;f7c814cf3c7e:37921 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T00:45:44,950 INFO [M:0;f7c814cf3c7e:37921 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T00:45:44,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T00:45:44,950 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T00:45:44,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:44,951 DEBUG [M:0;f7c814cf3c7e:37921 {}] zookeeper.ZKUtil(347): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T00:45:44,951 WARN [M:0;f7c814cf3c7e:37921 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T00:45:44,951 INFO [M:0;f7c814cf3c7e:37921 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/.lastflushedseqids 2024-12-06T00:45:44,964 INFO [M:0;f7c814cf3c7e:37921 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T00:45:44,964 INFO [M:0;f7c814cf3c7e:37921 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T00:45:44,965 DEBUG [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T00:45:44,965 INFO [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:44,965 DEBUG [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:44,965 DEBUG [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T00:45:44,965 DEBUG [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:44,965 INFO [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-12-06T00:45:44,976 DEBUG [M:0;f7c814cf3c7e:37921 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dfc9e9a808d44f27befb1daee1d88ba7 2024-12-06T00:45:44,994 DEBUG [M:0;f7c814cf3c7e:37921 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dfc9e9a808d44f27befb1daee1d88ba7 is 82, key is hbase:meta,,1/info:regioninfo/1733445942005/Put/seqid=0 2024-12-06T00:45:44,995 INFO [M:0;f7c814cf3c7e:37921 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dfc9e9a808d44f27befb1daee1d88ba7 2024-12-06T00:45:45,005 DEBUG [M:0;f7c814cf3c7e:37921 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a8e2044fe4cd4ff78c04206de061531e 2024-12-06T00:45:45,021 DEBUG [M:0;f7c814cf3c7e:37921 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a8e2044fe4cd4ff78c04206de061531e is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733445942751/Put/seqid=0 2024-12-06T00:45:45,022 INFO [M:0;f7c814cf3c7e:37921 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a8e2044fe4cd4ff78c04206de061531e 2024-12-06T00:45:45,032 DEBUG [M:0;f7c814cf3c7e:37921 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8d2d45aeceb8400ea68484e004a87975 2024-12-06T00:45:45,045 DEBUG [M:0;f7c814cf3c7e:37921 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8d2d45aeceb8400ea68484e004a87975 is 69, key is f7c814cf3c7e,39163,1733445940068/rs:state/1733445941309/Put/seqid=0 2024-12-06T00:45:45,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:45,045 INFO [RS:1;f7c814cf3c7e:39163 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T00:45:45,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39163-0x10064c9a73c0002, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:45,045 INFO [RS:1;f7c814cf3c7e:39163 {}] regionserver.HRegionServer(1031): Exiting; stopping=f7c814cf3c7e,39163,1733445940068; zookeeper connection closed. 2024-12-06T00:45:45,046 INFO [M:0;f7c814cf3c7e:37921 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8d2d45aeceb8400ea68484e004a87975 2024-12-06T00:45:45,046 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1c717a81 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1c717a81 2024-12-06T00:45:45,046 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-06T00:45:45,047 DEBUG [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dfc9e9a808d44f27befb1daee1d88ba7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dfc9e9a808d44f27befb1daee1d88ba7 2024-12-06T00:45:45,049 INFO [M:0;f7c814cf3c7e:37921 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dfc9e9a808d44f27befb1daee1d88ba7, entries=8, sequenceid=84, filesize=5.5 K 2024-12-06T00:45:45,050 DEBUG [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a8e2044fe4cd4ff78c04206de061531e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a8e2044fe4cd4ff78c04206de061531e 2024-12-06T00:45:45,052 INFO [M:0;f7c814cf3c7e:37921 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a8e2044fe4cd4ff78c04206de061531e, entries=9, sequenceid=84, filesize=6.5 K 2024-12-06T00:45:45,052 DEBUG [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8d2d45aeceb8400ea68484e004a87975 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8d2d45aeceb8400ea68484e004a87975 2024-12-06T00:45:45,054 INFO [M:0;f7c814cf3c7e:37921 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8d2d45aeceb8400ea68484e004a87975, entries=2, sequenceid=84, filesize=5.1 K 2024-12-06T00:45:45,055 INFO [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false 2024-12-06T00:45:45,057 INFO [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:45,057 DEBUG [M:0;f7c814cf3c7e:37921 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733445944965Disabling compacts and flushes for region at 1733445944965Disabling writes for close at 1733445944965Obtaining lock to block concurrent updates at 1733445944965Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733445944965Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733445944966 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733445944966Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733445944967 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733445944994 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733445944994Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733445944998 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733445945020 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733445945020Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733445945025 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733445945044 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733445945044Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7171b6d8: reopening flushed file at 1733445945047 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e23ce5e: reopening flushed file at 1733445945049 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e6ab957: reopening flushed file at 1733445945052 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false at 1733445945055 (+3 ms)Writing region close event to WAL at 1733445945057 (+2 ms)Closed at 1733445945057 2024-12-06T00:45:45,059 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T00:45:45,059 INFO [M:0;f7c814cf3c7e:37921 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T00:45:45,059 INFO [M:0;f7c814cf3c7e:37921 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37921 2024-12-06T00:45:45,059 INFO [M:0;f7c814cf3c7e:37921 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T00:45:45,162 INFO [M:0;f7c814cf3c7e:37921 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T00:45:45,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:45,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37921-0x10064c9a73c0000, quorum=127.0.0.1:50862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:45,167 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T00:45:45,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-06T00:45:45,188 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:45,197 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/f7c814cf3c7e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f7c814cf3c7e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733445943126 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733445944825 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f7c814cf3c7e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f7c814cf3c7e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1733445944821 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733445944656 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1148 (was 1321), ProcessCount=11 (was 11), AvailableMemoryMB=11618 (was 11964) 2024-12-06T00:45:45,203 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=1148, ProcessCount=11, AvailableMemoryMB=11618 2024-12-06T00:45:45,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/hadoop-log-dir so I do NOT create it in target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6 2024-12-06T00:45:45,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/hadoop-tmp-dir so I do NOT create it in target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6 2024-12-06T00:45:45,204 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:45,207 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/cluster_50c0365c-bd29-35f2-cbc3-980d581cfac2, deleteOnExit=true 2024-12-06T00:45:45,210 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/cluster_50c0365c-bd29-35f2-cbc3-980d581cfac2/zookeeper_0, clientPort=49718, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/cluster_50c0365c-bd29-35f2-cbc3-980d581cfac2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/cluster_50c0365c-bd29-35f2-cbc3-980d581cfac2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T00:45:45,211 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49718 2024-12-06T00:45:45,224 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 with version=8 2024-12-06T00:45:45,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/hbase-staging 2024-12-06T00:45:45,227 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f7c814cf3c7e:0 server-side Connection retries=45 2024-12-06T00:45:45,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:45,228 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:45,228 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T00:45:45,228 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:45,228 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T00:45:45,228 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T00:45:45,228 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T00:45:45,229 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43179 2024-12-06T00:45:45,230 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43179 connecting to ZooKeeper ensemble=127.0.0.1:49718 2024-12-06T00:45:45,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:431790x0, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T00:45:45,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43179-0x10064c9c0600000 connected 2024-12-06T00:45:45,252 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:45,252 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:45,253 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:45,254 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7, hbase.cluster.distributed=false 2024-12-06T00:45:45,256 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T00:45:45,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43179 2024-12-06T00:45:45,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43179 2024-12-06T00:45:45,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43179 2024-12-06T00:45:45,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43179 2024-12-06T00:45:45,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43179 2024-12-06T00:45:45,275 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f7c814cf3c7e:0 server-side Connection retries=45 2024-12-06T00:45:45,275 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:45,275 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:45,275 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T00:45:45,275 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:45,275 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T00:45:45,275 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T00:45:45,276 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T00:45:45,277 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43351 2024-12-06T00:45:45,279 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43351 connecting to ZooKeeper ensemble=127.0.0.1:49718 2024-12-06T00:45:45,279 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:45,279 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:45,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:433510x0, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T00:45:45,283 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43351-0x10064c9c0600001 connected 2024-12-06T00:45:45,283 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:45,284 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T00:45:45,284 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.cache 2024-12-06T00:45:45,284 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T00:45:45,285 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T00:45:45,285 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T00:45:45,285 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T00:45:45,285 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T00:45:45,285 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T00:45:45,286 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T00:45:45,286 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.persistence is complete. 2024-12-06T00:45:45,287 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T00:45:45,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T00:45:45,289 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T00:45:45,296 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43351 2024-12-06T00:45:45,296 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43351 2024-12-06T00:45:45,296 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43351 2024-12-06T00:45:45,297 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43351 2024-12-06T00:45:45,297 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43351 2024-12-06T00:45:45,313 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f7c814cf3c7e:0 server-side Connection retries=45 2024-12-06T00:45:45,313 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:45,313 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:45,313 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T00:45:45,313 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:45,313 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T00:45:45,313 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T00:45:45,313 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T00:45:45,314 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40783 2024-12-06T00:45:45,316 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40783 connecting to ZooKeeper ensemble=127.0.0.1:49718 2024-12-06T00:45:45,316 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:45,316 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:45,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407830x0, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T00:45:45,320 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40783-0x10064c9c0600002 connected 2024-12-06T00:45:45,320 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:45,320 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T00:45:45,323 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.cache 2024-12-06T00:45:45,323 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T00:45:45,323 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T00:45:45,323 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T00:45:45,324 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T00:45:45,324 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T00:45:45,324 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T00:45:45,324 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T00:45:45,324 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.persistence is complete. 2024-12-06T00:45:45,326 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T00:45:45,327 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T00:45:45,328 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T00:45:45,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40783 2024-12-06T00:45:45,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40783 2024-12-06T00:45:45,335 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40783 2024-12-06T00:45:45,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40783 2024-12-06T00:45:45,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40783 2024-12-06T00:45:45,352 DEBUG [M:0;f7c814cf3c7e:43179 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f7c814cf3c7e:43179 2024-12-06T00:45:45,352 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:45,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:45,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:45,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:45,356 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:45,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T00:45:45,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T00:45:45,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,359 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T00:45:45,359 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f7c814cf3c7e,43179,1733445945227 from backup master directory 2024-12-06T00:45:45,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:45,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:45,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:45,361 WARN [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T00:45:45,361 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:45,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T00:45:45,363 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/hbase.id] with ID: 508f74b7-6e50-406e-bd39-182bbdf6cb7f 2024-12-06T00:45:45,363 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/.tmp/hbase.id 2024-12-06T00:45:45,367 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/hbase.id] 2024-12-06T00:45:45,410 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:45,410 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T00:45:45,412 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-06T00:45:45,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,421 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T00:45:45,422 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T00:45:45,423 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T00:45:45,446 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store 2024-12-06T00:45:45,453 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:45,453 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T00:45:45,453 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:45,453 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:45,453 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T00:45:45,453 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:45,453 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:45,453 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733445945453Disabling compacts and flushes for region at 1733445945453Disabling writes for close at 1733445945453Writing region close event to WAL at 1733445945453Closed at 1733445945453 2024-12-06T00:45:45,454 WARN [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/.initializing 2024-12-06T00:45:45,454 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/WALs/f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:45,460 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T00:45:45,464 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f7c814cf3c7e%2C43179%2C1733445945227, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/WALs/f7c814cf3c7e,43179,1733445945227, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/oldWALs, maxLogs=10 2024-12-06T00:45:45,483 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/WALs/f7c814cf3c7e,43179,1733445945227/f7c814cf3c7e%2C43179%2C1733445945227.1733445945464 2024-12-06T00:45:45,483 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T00:45:45,483 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T00:45:45,483 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:45,484 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,484 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,490 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T00:45:45,491 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:45,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:45,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,495 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T00:45:45,495 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:45,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:45,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,500 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T00:45:45,500 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:45,500 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:45,500 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T00:45:45,504 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:45,505 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:45,505 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,505 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,505 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,506 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,506 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,506 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T00:45:45,506 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T00:45:45,513 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T00:45:45,513 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70928880, jitterRate=0.05692267417907715}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T00:45:45,514 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733445945484Initializing all the Stores at 1733445945484Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445945484Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445945484Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445945484Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445945484Cleaning up temporary data from old regions at 1733445945506 (+22 ms)Region opened successfully at 1733445945514 (+8 ms) 2024-12-06T00:45:45,515 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T00:45:45,519 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5708eb21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7c814cf3c7e/172.17.0.2:0 2024-12-06T00:45:45,520 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T00:45:45,520 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T00:45:45,520 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T00:45:45,520 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T00:45:45,521 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-06T00:45:45,521 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-06T00:45:45,521 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T00:45:45,523 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T00:45:45,524 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T00:45:45,525 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T00:45:45,526 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T00:45:45,526 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T00:45:45,528 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T00:45:45,528 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T00:45:45,529 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T00:45:45,530 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T00:45:45,532 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T00:45:45,533 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T00:45:45,535 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T00:45:45,537 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T00:45:45,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:45,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:45,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:45,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,540 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f7c814cf3c7e,43179,1733445945227, sessionid=0x10064c9c0600000, setting cluster-up flag (Was=false) 2024-12-06T00:45:45,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,549 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T00:45:45,550 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:45,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:45,559 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T00:45:45,561 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:45,561 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T00:45:45,563 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T00:45:45,563 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T00:45:45,563 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T00:45:45,564 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f7c814cf3c7e,43179,1733445945227 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T00:45:45,565 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f7c814cf3c7e:0, corePoolSize=5, maxPoolSize=5 2024-12-06T00:45:45,565 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f7c814cf3c7e:0, corePoolSize=5, maxPoolSize=5 2024-12-06T00:45:45,565 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f7c814cf3c7e:0, corePoolSize=5, maxPoolSize=5 2024-12-06T00:45:45,566 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f7c814cf3c7e:0, corePoolSize=5, maxPoolSize=5 2024-12-06T00:45:45,566 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f7c814cf3c7e:0, corePoolSize=10, maxPoolSize=10 2024-12-06T00:45:45,566 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,566 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f7c814cf3c7e:0, corePoolSize=2, maxPoolSize=2 2024-12-06T00:45:45,566 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,567 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733445975567 2024-12-06T00:45:45,567 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T00:45:45,567 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T00:45:45,567 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T00:45:45,567 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T00:45:45,567 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T00:45:45,567 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T00:45:45,567 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T00:45:45,568 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T00:45:45,568 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,568 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:45,568 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T00:45:45,569 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T00:45:45,569 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T00:45:45,568 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T00:45:45,569 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T00:45:45,569 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T00:45:45,572 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.large.0-1733445945569,5,FailOnTimeoutGroup] 2024-12-06T00:45:45,572 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.small.0-1733445945572,5,FailOnTimeoutGroup] 2024-12-06T00:45:45,572 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,572 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T00:45:45,572 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,572 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,591 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T00:45:45,591 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 2024-12-06T00:45:45,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:45,600 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T00:45:45,604 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T00:45:45,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:45,604 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:45,604 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T00:45:45,608 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T00:45:45,608 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:45,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:45,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T00:45:45,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T00:45:45,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:45,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:45,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T00:45:45,617 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T00:45:45,617 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:45,617 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:45,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T00:45:45,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740 2024-12-06T00:45:45,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740 2024-12-06T00:45:45,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T00:45:45,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T00:45:45,618 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T00:45:45,619 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T00:45:45,626 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T00:45:45,626 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70483767, jitterRate=0.05028997361660004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T00:45:45,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733445945599Initializing all the Stores at 1733445945599Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445945599Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445945599Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445945600 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445945600Cleaning up temporary data from old regions at 1733445945618 (+18 ms)Region opened successfully at 1733445945627 (+9 ms) 2024-12-06T00:45:45,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T00:45:45,627 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T00:45:45,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T00:45:45,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T00:45:45,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T00:45:45,628 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T00:45:45,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733445945627Disabling compacts and flushes for region at 1733445945627Disabling writes for close at 1733445945627Writing region close event to WAL at 1733445945628 (+1 ms)Closed at 1733445945628 2024-12-06T00:45:45,629 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T00:45:45,629 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T00:45:45,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T00:45:45,631 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T00:45:45,631 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T00:45:45,642 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(746): ClusterId : 508f74b7-6e50-406e-bd39-182bbdf6cb7f 2024-12-06T00:45:45,642 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(746): ClusterId : 508f74b7-6e50-406e-bd39-182bbdf6cb7f 2024-12-06T00:45:45,642 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T00:45:45,642 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T00:45:45,645 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T00:45:45,645 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T00:45:45,645 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T00:45:45,645 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T00:45:45,647 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T00:45:45,647 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T00:45:45,648 DEBUG [RS:0;f7c814cf3c7e:43351 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f6e6c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7c814cf3c7e/172.17.0.2:0 2024-12-06T00:45:45,648 DEBUG [RS:1;f7c814cf3c7e:40783 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ee16b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7c814cf3c7e/172.17.0.2:0 2024-12-06T00:45:45,664 DEBUG [RS:1;f7c814cf3c7e:40783 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f7c814cf3c7e:40783 2024-12-06T00:45:45,664 DEBUG [RS:0;f7c814cf3c7e:43351 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f7c814cf3c7e:43351 2024-12-06T00:45:45,664 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T00:45:45,664 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T00:45:45,664 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T00:45:45,664 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T00:45:45,664 DEBUG [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T00:45:45,664 DEBUG [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T00:45:45,665 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7c814cf3c7e,43179,1733445945227 with port=40783, startcode=1733445945313 2024-12-06T00:45:45,665 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7c814cf3c7e,43179,1733445945227 with port=43351, startcode=1733445945274 2024-12-06T00:45:45,666 DEBUG [RS:0;f7c814cf3c7e:43351 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T00:45:45,666 DEBUG [RS:1;f7c814cf3c7e:40783 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T00:45:45,669 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59161, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T00:45:45,669 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44817, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T00:45:45,670 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43179 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:45,670 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43179 {}] master.ServerManager(517): Registering regionserver=f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:45,671 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43179 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f7c814cf3c7e,43351,1733445945274 2024-12-06T00:45:45,671 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43179 {}] master.ServerManager(517): Registering regionserver=f7c814cf3c7e,43351,1733445945274 2024-12-06T00:45:45,671 DEBUG [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 2024-12-06T00:45:45,671 DEBUG [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T00:45:45,671 DEBUG [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T00:45:45,673 DEBUG [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 2024-12-06T00:45:45,673 DEBUG [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T00:45:45,673 DEBUG [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T00:45:45,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T00:45:45,678 DEBUG [RS:1;f7c814cf3c7e:40783 {}] zookeeper.ZKUtil(111): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:45,678 WARN [RS:1;f7c814cf3c7e:40783 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T00:45:45,678 INFO [RS:1;f7c814cf3c7e:40783 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T00:45:45,678 DEBUG [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:45,679 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f7c814cf3c7e,43351,1733445945274] 2024-12-06T00:45:45,679 DEBUG [RS:0;f7c814cf3c7e:43351 {}] zookeeper.ZKUtil(111): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f7c814cf3c7e,43351,1733445945274 2024-12-06T00:45:45,679 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f7c814cf3c7e,40783,1733445945313] 2024-12-06T00:45:45,679 WARN [RS:0;f7c814cf3c7e:43351 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T00:45:45,679 INFO [RS:0;f7c814cf3c7e:43351 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T00:45:45,679 DEBUG [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,43351,1733445945274 2024-12-06T00:45:45,687 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T00:45:45,689 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T00:45:45,689 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T00:45:45,690 INFO [RS:1;f7c814cf3c7e:40783 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T00:45:45,690 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,690 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T00:45:45,691 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T00:45:45,691 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,691 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,691 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,691 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T00:45:45,691 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,691 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,691 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,692 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f7c814cf3c7e:0, corePoolSize=2, maxPoolSize=2 2024-12-06T00:45:45,692 INFO [RS:0;f7c814cf3c7e:43351 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T00:45:45,692 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,692 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,692 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,692 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,692 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,692 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,692 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,692 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f7c814cf3c7e:0, corePoolSize=3, maxPoolSize=3 2024-12-06T00:45:45,692 DEBUG [RS:1;f7c814cf3c7e:40783 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f7c814cf3c7e:0, corePoolSize=3, maxPoolSize=3 2024-12-06T00:45:45,696 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T00:45:45,696 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,696 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,696 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,696 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T00:45:45,696 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,696 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,696 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,696 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,40783,1733445945313-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T00:45:45,696 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,697 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,697 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,697 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,697 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,697 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f7c814cf3c7e:0, corePoolSize=2, maxPoolSize=2 2024-12-06T00:45:45,697 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,697 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,697 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,697 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,697 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,698 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:45,698 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f7c814cf3c7e:0, corePoolSize=3, maxPoolSize=3 2024-12-06T00:45:45,698 DEBUG [RS:0;f7c814cf3c7e:43351 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f7c814cf3c7e:0, corePoolSize=3, maxPoolSize=3 2024-12-06T00:45:45,700 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,700 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,700 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,700 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,700 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,700 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,43351,1733445945274-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T00:45:45,712 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T00:45:45,712 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,40783,1733445945313-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,712 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,712 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.Replication(171): f7c814cf3c7e,40783,1733445945313 started 2024-12-06T00:45:45,714 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T00:45:45,715 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,43351,1733445945274-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,715 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,715 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.Replication(171): f7c814cf3c7e,43351,1733445945274 started 2024-12-06T00:45:45,727 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,727 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(1482): Serving as f7c814cf3c7e,40783,1733445945313, RpcServer on f7c814cf3c7e/172.17.0.2:40783, sessionid=0x10064c9c0600002 2024-12-06T00:45:45,727 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T00:45:45,727 DEBUG [RS:1;f7c814cf3c7e:40783 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:45,727 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7c814cf3c7e,40783,1733445945313' 2024-12-06T00:45:45,727 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T00:45:45,728 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T00:45:45,729 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T00:45:45,729 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T00:45:45,729 DEBUG [RS:1;f7c814cf3c7e:40783 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:45,729 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7c814cf3c7e,40783,1733445945313' 2024-12-06T00:45:45,729 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T00:45:45,729 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T00:45:45,730 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:45,730 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(1482): Serving as f7c814cf3c7e,43351,1733445945274, RpcServer on f7c814cf3c7e/172.17.0.2:43351, sessionid=0x10064c9c0600001 2024-12-06T00:45:45,730 DEBUG [RS:1;f7c814cf3c7e:40783 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T00:45:45,730 INFO [RS:1;f7c814cf3c7e:40783 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T00:45:45,730 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T00:45:45,730 INFO [RS:1;f7c814cf3c7e:40783 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T00:45:45,730 DEBUG [RS:0;f7c814cf3c7e:43351 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f7c814cf3c7e,43351,1733445945274 2024-12-06T00:45:45,730 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7c814cf3c7e,43351,1733445945274' 2024-12-06T00:45:45,730 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T00:45:45,731 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T00:45:45,731 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T00:45:45,731 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T00:45:45,731 DEBUG [RS:0;f7c814cf3c7e:43351 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f7c814cf3c7e,43351,1733445945274 2024-12-06T00:45:45,732 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7c814cf3c7e,43351,1733445945274' 2024-12-06T00:45:45,732 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T00:45:45,732 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T00:45:45,733 DEBUG [RS:0;f7c814cf3c7e:43351 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T00:45:45,733 INFO [RS:0;f7c814cf3c7e:43351 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T00:45:45,733 INFO [RS:0;f7c814cf3c7e:43351 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T00:45:45,782 WARN [f7c814cf3c7e:43179 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T00:45:45,831 INFO [RS:1;f7c814cf3c7e:40783 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T00:45:45,832 INFO [RS:1;f7c814cf3c7e:40783 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7c814cf3c7e%2C40783%2C1733445945313, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,40783,1733445945313, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs, maxLogs=55 2024-12-06T00:45:45,834 INFO [RS:0;f7c814cf3c7e:43351 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T00:45:45,835 INFO [RS:0;f7c814cf3c7e:43351 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7c814cf3c7e%2C43351%2C1733445945274, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,43351,1733445945274, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs, maxLogs=55 2024-12-06T00:45:45,851 INFO [RS:1;f7c814cf3c7e:40783 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,40783,1733445945313/f7c814cf3c7e%2C40783%2C1733445945313.1733445945833 2024-12-06T00:45:45,851 DEBUG [RS:1;f7c814cf3c7e:40783 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T00:45:45,860 INFO [RS:0;f7c814cf3c7e:43351 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,43351,1733445945274/f7c814cf3c7e%2C43351%2C1733445945274.1733445945836 2024-12-06T00:45:45,860 DEBUG [RS:0;f7c814cf3c7e:43351 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T00:45:46,032 DEBUG [f7c814cf3c7e:43179 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-06T00:45:46,032 DEBUG [f7c814cf3c7e:43179 {}] balancer.BalancerClusterState(204): Hosts are {f7c814cf3c7e=0} racks are {/default-rack=0} 2024-12-06T00:45:46,033 DEBUG [f7c814cf3c7e:43179 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T00:45:46,033 DEBUG [f7c814cf3c7e:43179 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T00:45:46,033 DEBUG [f7c814cf3c7e:43179 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T00:45:46,033 DEBUG [f7c814cf3c7e:43179 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T00:45:46,033 INFO [f7c814cf3c7e:43179 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T00:45:46,034 INFO [f7c814cf3c7e:43179 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T00:45:46,034 DEBUG [f7c814cf3c7e:43179 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T00:45:46,034 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:46,035 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f7c814cf3c7e,40783,1733445945313, state=OPENING 2024-12-06T00:45:46,037 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T00:45:46,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:46,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:46,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:46,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:46,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:46,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:46,040 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T00:45:46,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f7c814cf3c7e,40783,1733445945313}] 2024-12-06T00:45:46,193 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T00:45:46,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59807, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T00:45:46,198 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T00:45:46,198 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T00:45:46,199 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T00:45:46,200 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7c814cf3c7e%2C40783%2C1733445945313.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,40783,1733445945313, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs, maxLogs=55 2024-12-06T00:45:46,225 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,40783,1733445945313/f7c814cf3c7e%2C40783%2C1733445945313.meta.1733445946201.meta 2024-12-06T00:45:46,226 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T00:45:46,226 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T00:45:46,226 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T00:45:46,227 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T00:45:46,227 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T00:45:46,227 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T00:45:46,227 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:46,227 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T00:45:46,227 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T00:45:46,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T00:45:46,229 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T00:45:46,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:46,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:46,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T00:45:46,230 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T00:45:46,230 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:46,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:46,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T00:45:46,231 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T00:45:46,231 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:46,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:46,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T00:45:46,232 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T00:45:46,232 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:46,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T00:45:46,232 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T00:45:46,233 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740 2024-12-06T00:45:46,233 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740 2024-12-06T00:45:46,234 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T00:45:46,234 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T00:45:46,234 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T00:45:46,235 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T00:45:46,235 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62481796, jitterRate=-0.06894868612289429}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T00:45:46,235 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T00:45:46,236 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733445946227Writing region info on filesystem at 1733445946227Initializing all the Stores at 1733445946228 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445946228Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445946228Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445946228Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733445946228Cleaning up temporary data from old regions at 1733445946234 (+6 ms)Running coprocessor post-open hooks at 1733445946235 (+1 ms)Region opened successfully at 1733445946236 (+1 ms) 2024-12-06T00:45:46,237 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733445946193 2024-12-06T00:45:46,240 DEBUG [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T00:45:46,240 INFO [RS_OPEN_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T00:45:46,240 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:46,241 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f7c814cf3c7e,40783,1733445945313, state=OPEN 2024-12-06T00:45:46,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T00:45:46,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T00:45:46,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T00:45:46,243 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:46,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:46,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:46,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T00:45:46,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T00:45:46,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f7c814cf3c7e,40783,1733445945313 in 203 msec 2024-12-06T00:45:46,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T00:45:46,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 616 msec 2024-12-06T00:45:46,249 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T00:45:46,249 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T00:45:46,252 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T00:45:46,252 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f7c814cf3c7e,40783,1733445945313, seqNum=-1] 2024-12-06T00:45:46,253 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T00:45:46,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60365, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T00:45:46,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 696 msec 2024-12-06T00:45:46,260 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733445946259, completionTime=-1 2024-12-06T00:45:46,260 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T00:45:46,260 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T00:45:46,266 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T00:45:46,266 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733446006266 2024-12-06T00:45:46,266 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733446066266 2024-12-06T00:45:46,266 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-06T00:45:46,267 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-06T00:45:46,268 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,43179,1733445945227-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:46,268 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,43179,1733445945227-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:46,268 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,43179,1733445945227-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:46,268 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f7c814cf3c7e:43179, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:46,268 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:46,268 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:46,272 DEBUG [master/f7c814cf3c7e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T00:45:46,275 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.914sec 2024-12-06T00:45:46,275 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T00:45:46,275 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T00:45:46,275 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T00:45:46,275 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T00:45:46,275 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T00:45:46,275 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,43179,1733445945227-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T00:45:46,276 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,43179,1733445945227-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T00:45:46,276 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T00:45:46,276 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T00:45:46,277 INFO [master/f7c814cf3c7e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,43179,1733445945227-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:46,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T00:45:46,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f7c814cf3c7e,43179,-1 for getting cluster id 2024-12-06T00:45:46,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T00:45:46,345 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '508f74b7-6e50-406e-bd39-182bbdf6cb7f' 2024-12-06T00:45:46,345 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T00:45:46,346 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "508f74b7-6e50-406e-bd39-182bbdf6cb7f" 2024-12-06T00:45:46,346 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e1a8fa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T00:45:46,346 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f7c814cf3c7e,43179,-1] 2024-12-06T00:45:46,347 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T00:45:46,347 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:46,348 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T00:45:46,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T00:45:46,350 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T00:45:46,352 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f7c814cf3c7e,40783,1733445945313, seqNum=-1] 2024-12-06T00:45:46,352 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T00:45:46,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T00:45:46,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:46,357 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T00:45:46,358 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:46,358 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7255c9ea 2024-12-06T00:45:46,359 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T00:45:46,360 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T00:45:46,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43179 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733445946357', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T00:45:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43179 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733445946357 2024-12-06T00:45:46,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733445946357 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T00:45:46,364 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:46,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43179 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733445946357" procId is: 4 2024-12-06T00:45:46,365 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733445946357 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T00:45:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43179 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T00:45:46,378 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733445946357-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eff6006b10247b5ff94e6c95c88d182f, NAME => 'testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733445946357', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 2024-12-06T00:45:46,386 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733445946357-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:46,386 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733445946357-pool-0 {}] regionserver.HRegion(1722): Closing eff6006b10247b5ff94e6c95c88d182f, disabling compactions & flushes 2024-12-06T00:45:46,386 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733445946357-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:46,386 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733445946357-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:46,386 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733445946357-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. after waiting 0 ms 2024-12-06T00:45:46,386 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733445946357-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:46,386 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733445946357-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:46,386 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733445946357-pool-0 {}] regionserver.HRegion(1676): Region close journal for eff6006b10247b5ff94e6c95c88d182f: Waiting for close lock at 1733445946386Disabling compacts and flushes for region at 1733445946386Disabling writes for close at 1733445946386Writing region close event to WAL at 1733445946386Closed at 1733445946386 2024-12-06T00:45:46,387 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733445946357 execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T00:45:46,388 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733445946388"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733445946388"}]},"ts":"1733445946388"} 2024-12-06T00:45:46,391 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T00:45:46,392 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733445946357 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T00:45:46,392 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733445946357","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733445946392"}]},"ts":"1733445946392"} 2024-12-06T00:45:46,395 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733445946357, state=ENABLING in hbase:meta 2024-12-06T00:45:46,395 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f7c814cf3c7e=0} racks are {/default-rack=0} 2024-12-06T00:45:46,396 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T00:45:46,396 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T00:45:46,396 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T00:45:46,396 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T00:45:46,396 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T00:45:46,396 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T00:45:46,396 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T00:45:46,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733445946357, region=eff6006b10247b5ff94e6c95c88d182f, ASSIGN}] 2024-12-06T00:45:46,398 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733445946357, region=eff6006b10247b5ff94e6c95c88d182f, ASSIGN 2024-12-06T00:45:46,399 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733445946357, region=eff6006b10247b5ff94e6c95c88d182f, ASSIGN; state=OFFLINE, location=f7c814cf3c7e,43351,1733445945274; forceNewPlan=false, retain=false 2024-12-06T00:45:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43179 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T00:45:46,550 INFO [f7c814cf3c7e:43179 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T00:45:46,550 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eff6006b10247b5ff94e6c95c88d182f, regionState=OPENING, regionLocation=f7c814cf3c7e,43351,1733445945274 2024-12-06T00:45:46,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733445946357, region=eff6006b10247b5ff94e6c95c88d182f, ASSIGN because future has completed 2024-12-06T00:45:46,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eff6006b10247b5ff94e6c95c88d182f, server=f7c814cf3c7e,43351,1733445945274}] 2024-12-06T00:45:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43179 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T00:45:46,707 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T00:45:46,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46669, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T00:45:46,711 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:46,711 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => eff6006b10247b5ff94e6c95c88d182f, NAME => 'testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f.', STARTKEY => '', ENDKEY => ''} 2024-12-06T00:45:46,712 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733445946357 eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,712 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:46,712 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,712 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,713 INFO [StoreOpener-eff6006b10247b5ff94e6c95c88d182f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,717 INFO [StoreOpener-eff6006b10247b5ff94e6c95c88d182f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eff6006b10247b5ff94e6c95c88d182f columnFamilyName family 2024-12-06T00:45:46,717 DEBUG [StoreOpener-eff6006b10247b5ff94e6c95c88d182f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:46,717 INFO [StoreOpener-eff6006b10247b5ff94e6c95c88d182f-1 {}] regionserver.HStore(327): Store=eff6006b10247b5ff94e6c95c88d182f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:46,717 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,718 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,718 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,718 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,718 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,719 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,725 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T00:45:46,726 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened eff6006b10247b5ff94e6c95c88d182f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69756299, jitterRate=0.03944985568523407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T00:45:46,726 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:46,727 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for eff6006b10247b5ff94e6c95c88d182f: Running coprocessor pre-open hook at 1733445946712Writing region info on filesystem at 1733445946712Initializing all the Stores at 1733445946712Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445946712Cleaning up temporary data from old regions at 1733445946718 (+6 ms)Running coprocessor post-open hooks at 1733445946726 (+8 ms)Region opened successfully at 1733445946727 (+1 ms) 2024-12-06T00:45:46,727 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f., pid=6, masterSystemTime=1733445946706 2024-12-06T00:45:46,730 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:46,730 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:46,731 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eff6006b10247b5ff94e6c95c88d182f, regionState=OPEN, openSeqNum=2, regionLocation=f7c814cf3c7e,43351,1733445945274 2024-12-06T00:45:46,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eff6006b10247b5ff94e6c95c88d182f, server=f7c814cf3c7e,43351,1733445945274 because future has completed 2024-12-06T00:45:46,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T00:45:46,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure eff6006b10247b5ff94e6c95c88d182f, server=f7c814cf3c7e,43351,1733445945274 in 180 msec 2024-12-06T00:45:46,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T00:45:46,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733445946357, region=eff6006b10247b5ff94e6c95c88d182f, ASSIGN in 340 msec 2024-12-06T00:45:46,739 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733445946357 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T00:45:46,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733445946357","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733445946739"}]},"ts":"1733445946739"} 2024-12-06T00:45:46,741 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733445946357, state=ENABLED in hbase:meta 2024-12-06T00:45:46,742 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733445946357 execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T00:45:46,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733445946357 in 380 msec 2024-12-06T00:45:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43179 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T00:45:46,997 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733445946357 completed 2024-12-06T00:45:46,997 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733445946357 get assigned. Timeout = 60000ms 2024-12-06T00:45:46,997 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T00:45:47,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733445946357 assigned to meta. Checking AM states. 2024-12-06T00:45:47,001 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T00:45:47,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733445946357 assigned. 2024-12-06T00:45:47,004 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733445946357', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f., hostname=f7c814cf3c7e,43351,1733445945274, seqNum=2] 2024-12-06T00:45:47,004 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T00:45:47,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T00:45:47,011 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eff6006b10247b5ff94e6c95c88d182f 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-06T00:45:47,023 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f8562d1fff33455787770472bc3d00ea 2024-12-06T00:45:47,036 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/.tmp/family/f8562d1fff33455787770472bc3d00ea is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-06T00:45:47,037 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/.tmp/family/f8562d1fff33455787770472bc3d00ea 2024-12-06T00:45:47,039 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/.tmp/family/f8562d1fff33455787770472bc3d00ea as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea 2024-12-06T00:45:47,040 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea, delay=933 ms 2024-12-06T00:45:47,042 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea, entries=2, sequenceid=6, filesize=4.9 K 2024-12-06T00:45:47,042 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for eff6006b10247b5ff94e6c95c88d182f in 31ms, sequenceid=6, compaction requested=false 2024-12-06T00:45:47,043 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733445946357' 2024-12-06T00:45:47,043 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for eff6006b10247b5ff94e6c95c88d182f: 2024-12-06T00:45:47,473 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T00:45:47,975 DEBUG [hfile-prefetch-1733445944656 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f8562d1fff33455787770472bc3d00ea_0. 2024-12-06T00:45:47,975 DEBUG [hfile-prefetch-1733445944656 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f8562d1fff33455787770472bc3d00ea_117. 2024-12-06T00:45:47,976 DEBUG [hfile-prefetch-1733445944656 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea, with total blocks 2, and data blocks 1 2024-12-06T00:45:47,976 DEBUG [hfile-prefetch-1733445944656 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-06T00:45:47,976 DEBUG [hfile-prefetch-1733445944656 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-06T00:45:47,976 DEBUG [hfile-prefetch-1733445944656 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea. Will sleep 100ms and try the verification again. 2024-12-06T00:45:47,987 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-06T00:45:47,987 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-06T00:45:47,987 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T00:45:47,987 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T00:45:47,994 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-06T00:45:47,994 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733445947987 2024-12-06T00:45:47,994 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-06T00:45:48,044 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;f7c814cf3c7e:43351,5,FailOnTimeoutGroup] 2024-12-06T00:45:48,044 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f7c814cf3c7e,43351,1733445945274' ***** 2024-12-06T00:45:48,044 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-06T00:45:48,044 INFO [RS:0;f7c814cf3c7e:43351 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T00:45:48,045 INFO [RS:0;f7c814cf3c7e:43351 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.persistence 2024-12-06T00:45:48,045 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T00:45:48,045 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T00:45:48,045 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T00:45:48,045 INFO [RS:0;f7c814cf3c7e:43351 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T00:45:48,045 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T00:45:48,055 DEBUG [RS:0;f7c814cf3c7e:43351 {}] bucket.BucketCache(1487): Thread RS:0;f7c814cf3c7e:43351 started persisting bucket cache to file 2024-12-06T00:45:48,055 DEBUG [RS:0;f7c814cf3c7e:43351 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T00:45:48,055 DEBUG [RS:0;f7c814cf3c7e:43351 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T00:45:48,059 DEBUG [RS:0;f7c814cf3c7e:43351 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-06T00:45:48,059 DEBUG [RS:0;f7c814cf3c7e:43351 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733445948055 2024-12-06T00:45:48,059 DEBUG [RS:0;f7c814cf3c7e:43351 {}] bucket.BucketCache(1509): Thread RS:0;f7c814cf3c7e:43351 finished persisting bucket cache to file, renaming 2024-12-06T00:45:48,059 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T00:45:48,059 INFO [RS:0;f7c814cf3c7e:43351 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T00:45:48,059 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T00:45:48,059 INFO [RS:0;f7c814cf3c7e:43351 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T00:45:48,059 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(3091): Received CLOSE for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,060 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(959): stopping server f7c814cf3c7e,43351,1733445945274 2024-12-06T00:45:48,060 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T00:45:48,060 INFO [RS:0;f7c814cf3c7e:43351 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f7c814cf3c7e:43351. 2024-12-06T00:45:48,060 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T00:45:48,060 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eff6006b10247b5ff94e6c95c88d182f, disabling compactions & flushes 2024-12-06T00:45:48,060 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:48,060 DEBUG [RS:0;f7c814cf3c7e:43351 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T00:45:48,060 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:48,060 DEBUG [RS:0;f7c814cf3c7e:43351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:48,060 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. after waiting 0 ms 2024-12-06T00:45:48,060 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:48,060 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-06T00:45:48,060 DEBUG [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(1325): Online Regions={eff6006b10247b5ff94e6c95c88d182f=testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f.} 2024-12-06T00:45:48,061 DEBUG [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(1351): Waiting on eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,061 DEBUG [StoreFileCloser-eff6006b10247b5ff94e6c95c88d182f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea 2024-12-06T00:45:48,063 WARN [hfile-prefetch-1733445944656 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-06T00:45:48,064 DEBUG [hfile-prefetch-1733445944656 {}] hfile.PrefetchExecutor(121): Prefetch completed for f8562d1fff33455787770472bc3d00ea 2024-12-06T00:45:48,066 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-06T00:45:48,067 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:48,067 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eff6006b10247b5ff94e6c95c88d182f: Waiting for close lock at 1733445948060Running coprocessor pre-close hooks at 1733445948060Disabling compacts and flushes for region at 1733445948060Disabling writes for close at 1733445948060Writing region close event to WAL at 1733445948061 (+1 ms)Running coprocessor post-close hooks at 1733445948067 (+6 ms)Closed at 1733445948067 2024-12-06T00:45:48,067 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:48,102 INFO [regionserver/f7c814cf3c7e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T00:45:48,261 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(976): stopping server f7c814cf3c7e,43351,1733445945274; all regions closed. 2024-12-06T00:45:48,263 DEBUG [RS:0;f7c814cf3c7e:43351 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs 2024-12-06T00:45:48,263 INFO [RS:0;f7c814cf3c7e:43351 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7c814cf3c7e%2C43351%2C1733445945274:(num 1733445945836) 2024-12-06T00:45:48,263 DEBUG [RS:0;f7c814cf3c7e:43351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:48,263 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T00:45:48,263 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T00:45:48,264 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.ChoreService(370): Chore service for: regionserver/f7c814cf3c7e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T00:45:48,264 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T00:45:48,264 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T00:45:48,264 INFO [regionserver/f7c814cf3c7e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T00:45:48,264 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T00:45:48,264 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T00:45:48,264 INFO [RS:0;f7c814cf3c7e:43351 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43351 2024-12-06T00:45:48,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f7c814cf3c7e,43351,1733445945274 2024-12-06T00:45:48,269 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T00:45:48,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T00:45:48,271 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f7c814cf3c7e,43351,1733445945274] 2024-12-06T00:45:48,273 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f7c814cf3c7e,43351,1733445945274 already deleted, retry=false 2024-12-06T00:45:48,273 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f7c814cf3c7e,43351,1733445945274 on f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:48,278 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f7c814cf3c7e,43351,1733445945274, splitWal=true, meta=false 2024-12-06T00:45:48,280 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for f7c814cf3c7e,43351,1733445945274 (carryingMeta=false) f7c814cf3c7e,43351,1733445945274/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@53c9206c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-06T00:45:48,280 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f7c814cf3c7e,43351,1733445945274, splitWal=true, meta=false 2024-12-06T00:45:48,281 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): f7c814cf3c7e,43351,1733445945274 had 1 regions 2024-12-06T00:45:48,282 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f7c814cf3c7e,43351,1733445945274, splitWal=true, meta=false, isMeta: false 2024-12-06T00:45:48,282 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,43351,1733445945274-splitting 2024-12-06T00:45:48,283 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,43351,1733445945274-splitting dir is empty, no logs to split. 2024-12-06T00:45:48,283 INFO [PEWorker-2 {}] master.SplitWALManager(105): f7c814cf3c7e,43351,1733445945274 WAL count=0, meta=false 2024-12-06T00:45:48,284 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,43351,1733445945274-splitting dir is empty, no logs to split. 2024-12-06T00:45:48,284 INFO [PEWorker-2 {}] master.SplitWALManager(105): f7c814cf3c7e,43351,1733445945274 WAL count=0, meta=false 2024-12-06T00:45:48,284 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if f7c814cf3c7e,43351,1733445945274 WAL splitting is done? wals=0, meta=false 2024-12-06T00:45:48,285 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for f7c814cf3c7e,43351,1733445945274 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,43351,1733445945274-splitting does not exist 2024-12-06T00:45:48,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733445946357, region=eff6006b10247b5ff94e6c95c88d182f, ASSIGN}] 2024-12-06T00:45:48,287 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733445946357, region=eff6006b10247b5ff94e6c95c88d182f, ASSIGN 2024-12-06T00:45:48,288 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733445946357, region=eff6006b10247b5ff94e6c95c88d182f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-06T00:45:48,371 INFO [RS:0;f7c814cf3c7e:43351 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T00:45:48,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:48,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43351-0x10064c9c0600001, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:48,371 INFO [RS:0;f7c814cf3c7e:43351 {}] regionserver.HRegionServer(1031): Exiting; stopping=f7c814cf3c7e,43351,1733445945274; zookeeper connection closed. 2024-12-06T00:45:48,371 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7f3a7ae4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7f3a7ae4 2024-12-06T00:45:48,439 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=eff6006b10247b5ff94e6c95c88d182f, regionState=OPENING, regionLocation=f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:48,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733445946357, region=eff6006b10247b5ff94e6c95c88d182f, ASSIGN because future has completed 2024-12-06T00:45:48,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure eff6006b10247b5ff94e6c95c88d182f, server=f7c814cf3c7e,40783,1733445945313}] 2024-12-06T00:45:48,561 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f7c814cf3c7e:0 server-side Connection retries=45 2024-12-06T00:45:48,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:48,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:48,561 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T00:45:48,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T00:45:48,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T00:45:48,561 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T00:45:48,561 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T00:45:48,562 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39181 2024-12-06T00:45:48,564 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39181 connecting to ZooKeeper ensemble=127.0.0.1:49718 2024-12-06T00:45:48,564 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:48,565 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:48,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391810x0, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T00:45:48,569 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:391810x0, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-06T00:45:48,569 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39181-0x10064c9c0600003 connected 2024-12-06T00:45:48,569 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-06T00:45:48,569 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T00:45:48,570 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.cache 2024-12-06T00:45:48,570 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T00:45:48,570 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T00:45:48,570 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T00:45:48,571 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T00:45:48,571 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T00:45:48,572 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T00:45:48,572 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-06T00:45:48,572 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39181-0x10064c9c0600003, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T00:45:48,573 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-12-06T00:45:48,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39181-0x10064c9c0600003, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T00:45:48,575 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39181 2024-12-06T00:45:48,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39181 2024-12-06T00:45:48,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39181 2024-12-06T00:45:48,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39181 2024-12-06T00:45:48,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39181 2024-12-06T00:45:48,578 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(746): ClusterId : 508f74b7-6e50-406e-bd39-182bbdf6cb7f 2024-12-06T00:45:48,578 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T00:45:48,580 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T00:45:48,580 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T00:45:48,581 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-06T00:45:48,582 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T00:45:48,582 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-12-06T00:45:48,582 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.persistence is complete. 2024-12-06T00:45:48,583 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T00:45:48,583 DEBUG [RS:2;f7c814cf3c7e:39181 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117e512a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7c814cf3c7e/172.17.0.2:0 2024-12-06T00:45:48,597 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:48,598 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => eff6006b10247b5ff94e6c95c88d182f, NAME => 'testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f.', STARTKEY => '', ENDKEY => ''} 2024-12-06T00:45:48,598 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733445946357 eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,598 DEBUG [RS:2;f7c814cf3c7e:39181 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f7c814cf3c7e:39181 2024-12-06T00:45:48,598 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T00:45:48,598 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T00:45:48,598 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,598 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T00:45:48,598 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,598 DEBUG [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T00:45:48,599 INFO [StoreOpener-eff6006b10247b5ff94e6c95c88d182f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,599 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7c814cf3c7e,43179,1733445945227 with port=39181, startcode=1733445948560 2024-12-06T00:45:48,600 DEBUG [RS:2;f7c814cf3c7e:39181 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T00:45:48,600 INFO [StoreOpener-eff6006b10247b5ff94e6c95c88d182f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eff6006b10247b5ff94e6c95c88d182f columnFamilyName family 2024-12-06T00:45:48,600 DEBUG [StoreOpener-eff6006b10247b5ff94e6c95c88d182f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T00:45:48,602 DEBUG [StoreFileOpener-eff6006b10247b5ff94e6c95c88d182f-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea, delay=929 ms 2024-12-06T00:45:48,602 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52917, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T00:45:48,602 DEBUG [StoreOpener-eff6006b10247b5ff94e6c95c88d182f-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea 2024-12-06T00:45:48,603 INFO [StoreOpener-eff6006b10247b5ff94e6c95c88d182f-1 {}] regionserver.HStore(327): Store=eff6006b10247b5ff94e6c95c88d182f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T00:45:48,603 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,603 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43179 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f7c814cf3c7e,39181,1733445948560 2024-12-06T00:45:48,603 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43179 {}] master.ServerManager(517): Registering regionserver=f7c814cf3c7e,39181,1733445948560 2024-12-06T00:45:48,603 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,604 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,604 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,604 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,605 DEBUG [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7 2024-12-06T00:45:48,605 DEBUG [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T00:45:48,605 DEBUG [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T00:45:48,605 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,606 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened eff6006b10247b5ff94e6c95c88d182f; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69218947, jitterRate=0.031442686915397644}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T00:45:48,606 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:48,606 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for eff6006b10247b5ff94e6c95c88d182f: Running coprocessor pre-open hook at 1733445948599Writing region info on filesystem at 1733445948599Initializing all the Stores at 1733445948599Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733445948599Cleaning up temporary data from old regions at 1733445948604 (+5 ms)Running coprocessor post-open hooks at 1733445948606 (+2 ms)Region opened successfully at 1733445948606 2024-12-06T00:45:48,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T00:45:48,607 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f., pid=9, masterSystemTime=1733445948595 2024-12-06T00:45:48,608 DEBUG [RS:2;f7c814cf3c7e:39181 {}] zookeeper.ZKUtil(111): regionserver:39181-0x10064c9c0600003, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f7c814cf3c7e,39181,1733445948560 2024-12-06T00:45:48,608 WARN [RS:2;f7c814cf3c7e:39181 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T00:45:48,608 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f7c814cf3c7e,39181,1733445948560] 2024-12-06T00:45:48,608 INFO [RS:2;f7c814cf3c7e:39181 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T00:45:48,608 DEBUG [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39181,1733445948560 2024-12-06T00:45:48,610 DEBUG [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:48,610 INFO [RS_OPEN_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:48,610 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=eff6006b10247b5ff94e6c95c88d182f, regionState=OPEN, openSeqNum=10, regionLocation=f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:48,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure eff6006b10247b5ff94e6c95c88d182f, server=f7c814cf3c7e,40783,1733445945313 because future has completed 2024-12-06T00:45:48,615 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T00:45:48,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-06T00:45:48,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure eff6006b10247b5ff94e6c95c88d182f, server=f7c814cf3c7e,40783,1733445945313 in 173 msec 2024-12-06T00:45:48,618 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T00:45:48,618 INFO [RS:2;f7c814cf3c7e:39181 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T00:45:48,618 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:48,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T00:45:48,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733445946357, region=eff6006b10247b5ff94e6c95c88d182f, ASSIGN in 331 msec 2024-12-06T00:45:48,619 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server f7c814cf3c7e,43351,1733445945274 after splitting done 2024-12-06T00:45:48,620 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T00:45:48,620 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T00:45:48,621 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:48,621 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,621 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,621 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,621 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,621 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,621 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f7c814cf3c7e:0, corePoolSize=2, maxPoolSize=2 2024-12-06T00:45:48,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure f7c814cf3c7e,43351,1733445945274, splitWal=true, meta=false in 346 msec 2024-12-06T00:45:48,622 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,622 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,622 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,622 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,622 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,622 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f7c814cf3c7e:0, corePoolSize=1, maxPoolSize=1 2024-12-06T00:45:48,622 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f7c814cf3c7e:0, corePoolSize=3, maxPoolSize=3 2024-12-06T00:45:48,622 DEBUG [RS:2;f7c814cf3c7e:39181 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f7c814cf3c7e:0, corePoolSize=3, maxPoolSize=3 2024-12-06T00:45:48,624 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:48,624 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:48,624 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:48,624 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:48,624 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:48,624 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,39181,1733445948560-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T00:45:48,649 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T00:45:48,649 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7c814cf3c7e,39181,1733445948560-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:48,649 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:48,649 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.Replication(171): f7c814cf3c7e,39181,1733445948560 started 2024-12-06T00:45:48,665 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T00:45:48,665 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(1482): Serving as f7c814cf3c7e,39181,1733445948560, RpcServer on f7c814cf3c7e/172.17.0.2:39181, sessionid=0x10064c9c0600003 2024-12-06T00:45:48,665 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T00:45:48,665 DEBUG [RS:2;f7c814cf3c7e:39181 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f7c814cf3c7e,39181,1733445948560 2024-12-06T00:45:48,665 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7c814cf3c7e,39181,1733445948560' 2024-12-06T00:45:48,665 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T00:45:48,666 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T00:45:48,667 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T00:45:48,667 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T00:45:48,667 DEBUG [RS:2;f7c814cf3c7e:39181 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f7c814cf3c7e,39181,1733445948560 2024-12-06T00:45:48,667 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7c814cf3c7e,39181,1733445948560' 2024-12-06T00:45:48,667 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T00:45:48,668 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T00:45:48,668 DEBUG [RS:2;f7c814cf3c7e:39181 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T00:45:48,668 INFO [RS:2;f7c814cf3c7e:39181 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T00:45:48,669 INFO [RS:2;f7c814cf3c7e:39181 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T00:45:48,770 INFO [RS:2;f7c814cf3c7e:39181 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T00:45:48,771 INFO [RS:2;f7c814cf3c7e:39181 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7c814cf3c7e%2C39181%2C1733445948560, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39181,1733445948560, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs, maxLogs=55 2024-12-06T00:45:48,791 INFO [RS:2;f7c814cf3c7e:39181 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/WALs/f7c814cf3c7e,39181,1733445948560/f7c814cf3c7e%2C39181%2C1733445948560.1733445948771 2024-12-06T00:45:48,792 DEBUG [RS:2;f7c814cf3c7e:39181 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T00:45:49,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T00:45:49,165 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T00:45:49,165 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T00:45:49,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:49,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:49,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T00:45:49,166 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T00:45:49,166 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1126985514, stopped=false 2024-12-06T00:45:49,166 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f7c814cf3c7e,43179,1733445945227 2024-12-06T00:45:49,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39181-0x10064c9c0600003, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:49,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:49,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T00:45:49,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39181-0x10064c9c0600003, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:49,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T00:45:49,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:49,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:49,168 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T00:45:49,168 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T00:45:49,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:49,168 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f7c814cf3c7e,40783,1733445945313' ***** 2024-12-06T00:45:49,168 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T00:45:49,169 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f7c814cf3c7e,39181,1733445948560' ***** 2024-12-06T00:45:49,169 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T00:45:49,169 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:49,169 INFO [RS:1;f7c814cf3c7e:40783 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T00:45:49,169 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39181-0x10064c9c0600003, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:49,169 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T00:45:49,169 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T00:45:49,169 INFO [RS:1;f7c814cf3c7e:40783 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.persistence 2024-12-06T00:45:49,169 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T00:45:49,169 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T00:45:49,169 INFO [RS:1;f7c814cf3c7e:40783 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T00:45:49,170 INFO [RS:2;f7c814cf3c7e:39181 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T00:45:49,170 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T00:45:49,170 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T00:45:49,170 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T00:45:49,170 INFO [RS:2;f7c814cf3c7e:39181 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb48c42f-0162-50ab-0176-2ddd4dfadcb6/bucket.persistence 2024-12-06T00:45:49,170 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T00:45:49,170 INFO [RS:2;f7c814cf3c7e:39181 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T00:45:49,170 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T00:45:49,180 DEBUG [RS:1;f7c814cf3c7e:40783 {}] bucket.BucketCache(1487): Thread RS:1;f7c814cf3c7e:40783 started persisting bucket cache to file 2024-12-06T00:45:49,180 DEBUG [RS:1;f7c814cf3c7e:40783 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T00:45:49,180 DEBUG [RS:1;f7c814cf3c7e:40783 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T00:45:49,180 DEBUG [RS:2;f7c814cf3c7e:39181 {}] bucket.BucketCache(1487): Thread RS:2;f7c814cf3c7e:39181 started persisting bucket cache to file 2024-12-06T00:45:49,181 DEBUG [RS:2;f7c814cf3c7e:39181 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T00:45:49,181 DEBUG [RS:2;f7c814cf3c7e:39181 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T00:45:49,183 DEBUG [RS:1;f7c814cf3c7e:40783 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T00:45:49,183 DEBUG [RS:1;f7c814cf3c7e:40783 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733445949180 2024-12-06T00:45:49,183 DEBUG [RS:1;f7c814cf3c7e:40783 {}] bucket.BucketCache(1509): Thread RS:1;f7c814cf3c7e:40783 finished persisting bucket cache to file, renaming 2024-12-06T00:45:49,183 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T00:45:49,184 INFO [RS:1;f7c814cf3c7e:40783 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T00:45:49,184 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T00:45:49,184 INFO [RS:1;f7c814cf3c7e:40783 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T00:45:49,184 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(3091): Received CLOSE for eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:49,184 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(959): stopping server f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:49,184 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T00:45:49,184 INFO [RS:1;f7c814cf3c7e:40783 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f7c814cf3c7e:40783. 2024-12-06T00:45:49,184 DEBUG [RS:2;f7c814cf3c7e:39181 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-06T00:45:49,184 DEBUG [RS:2;f7c814cf3c7e:39181 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733445949180 2024-12-06T00:45:49,184 DEBUG [RS:1;f7c814cf3c7e:40783 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T00:45:49,184 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T00:45:49,184 DEBUG [RS:2;f7c814cf3c7e:39181 {}] bucket.BucketCache(1509): Thread RS:2;f7c814cf3c7e:39181 finished persisting bucket cache to file, renaming 2024-12-06T00:45:49,184 DEBUG [RS:1;f7c814cf3c7e:40783 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:49,184 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eff6006b10247b5ff94e6c95c88d182f, disabling compactions & flushes 2024-12-06T00:45:49,184 WARN [RS:2;f7c814cf3c7e:39181 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-06T00:45:49,184 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:49,184 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T00:45:49,184 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T00:45:49,184 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T00:45:49,184 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:49,185 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T00:45:49,185 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. after waiting 0 ms 2024-12-06T00:45:49,185 INFO [RS:2;f7c814cf3c7e:39181 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T00:45:49,185 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T00:45:49,185 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:49,185 INFO [RS:2;f7c814cf3c7e:39181 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T00:45:49,185 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(959): stopping server f7c814cf3c7e,39181,1733445948560 2024-12-06T00:45:49,185 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T00:45:49,185 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T00:45:49,185 INFO [RS:2;f7c814cf3c7e:39181 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f7c814cf3c7e:39181. 2024-12-06T00:45:49,185 DEBUG [RS:2;f7c814cf3c7e:39181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T00:45:49,185 DEBUG [RS:2;f7c814cf3c7e:39181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:49,185 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-06T00:45:49,185 DEBUG [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(1325): Online Regions={eff6006b10247b5ff94e6c95c88d182f=testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f., 1588230740=hbase:meta,,1.1588230740} 2024-12-06T00:45:49,185 DEBUG [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, eff6006b10247b5ff94e6c95c88d182f 2024-12-06T00:45:49,185 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T00:45:49,185 DEBUG [StoreFileCloser-eff6006b10247b5ff94e6c95c88d182f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/family/f8562d1fff33455787770472bc3d00ea 2024-12-06T00:45:49,185 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T00:45:49,186 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T00:45:49,186 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(976): stopping server f7c814cf3c7e,39181,1733445948560; all regions closed. 2024-12-06T00:45:49,186 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T00:45:49,186 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T00:45:49,186 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T00:45:49,187 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-06T00:45:49,188 DEBUG [RS:2;f7c814cf3c7e:39181 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs 2024-12-06T00:45:49,188 INFO [RS:2;f7c814cf3c7e:39181 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7c814cf3c7e%2C39181%2C1733445948560:(num 1733445948771) 2024-12-06T00:45:49,188 DEBUG [RS:2;f7c814cf3c7e:39181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:49,188 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T00:45:49,188 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T00:45:49,188 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.ChoreService(370): Chore service for: regionserver/f7c814cf3c7e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T00:45:49,188 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T00:45:49,188 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T00:45:49,188 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T00:45:49,188 INFO [regionserver/f7c814cf3c7e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T00:45:49,188 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T00:45:49,189 INFO [RS:2;f7c814cf3c7e:39181 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39181 2024-12-06T00:45:49,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39181-0x10064c9c0600003, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f7c814cf3c7e,39181,1733445948560 2024-12-06T00:45:49,191 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T00:45:49,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T00:45:49,193 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f7c814cf3c7e,39181,1733445948560] 2024-12-06T00:45:49,194 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f7c814cf3c7e,39181,1733445948560 already deleted, retry=false 2024-12-06T00:45:49,194 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f7c814cf3c7e,39181,1733445948560 expired; onlineServers=1 2024-12-06T00:45:49,199 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/default/testBlockEvictionOnGracefulStop1733445946357/eff6006b10247b5ff94e6c95c88d182f/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-06T00:45:49,199 INFO [regionserver/f7c814cf3c7e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T00:45:49,199 INFO [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:49,200 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eff6006b10247b5ff94e6c95c88d182f: Waiting for close lock at 1733445949184Running coprocessor pre-close hooks at 1733445949184Disabling compacts and flushes for region at 1733445949184Disabling writes for close at 1733445949185 (+1 ms)Writing region close event to WAL at 1733445949187 (+2 ms)Running coprocessor post-close hooks at 1733445949199 (+12 ms)Closed at 1733445949199 2024-12-06T00:45:49,200 DEBUG [RS_CLOSE_REGION-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f. 2024-12-06T00:45:49,201 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bda36d91dd454942adebf065334a7caa 2024-12-06T00:45:49,215 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/info/bda36d91dd454942adebf065334a7caa is 209, key is testBlockEvictionOnGracefulStop1733445946357,,1733445946361.eff6006b10247b5ff94e6c95c88d182f./info:regioninfo/1733445948610/Put/seqid=0 2024-12-06T00:45:49,216 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/info/bda36d91dd454942adebf065334a7caa 2024-12-06T00:45:49,224 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0c716a72bccc4e1e9b3bcc9b9a862c39 2024-12-06T00:45:49,228 INFO [regionserver/f7c814cf3c7e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T00:45:49,237 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/ns/0c716a72bccc4e1e9b3bcc9b9a862c39 is 43, key is default/ns:d/1733445946255/Put/seqid=0 2024-12-06T00:45:49,238 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/ns/0c716a72bccc4e1e9b3bcc9b9a862c39 2024-12-06T00:45:49,247 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 216f65a7b5ca4082b0a79c2703895415 2024-12-06T00:45:49,264 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/table/216f65a7b5ca4082b0a79c2703895415 is 80, key is testBlockEvictionOnGracefulStop1733445946357/table:state/1733445946739/Put/seqid=0 2024-12-06T00:45:49,265 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/table/216f65a7b5ca4082b0a79c2703895415 2024-12-06T00:45:49,267 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/info/bda36d91dd454942adebf065334a7caa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/info/bda36d91dd454942adebf065334a7caa 2024-12-06T00:45:49,269 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/info/bda36d91dd454942adebf065334a7caa, delay=935 ms 2024-12-06T00:45:49,271 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/info/bda36d91dd454942adebf065334a7caa, entries=14, sequenceid=13, filesize=7.5 K 2024-12-06T00:45:49,271 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/ns/0c716a72bccc4e1e9b3bcc9b9a862c39 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/ns/0c716a72bccc4e1e9b3bcc9b9a862c39 2024-12-06T00:45:49,273 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/ns/0c716a72bccc4e1e9b3bcc9b9a862c39, delay=980 ms 2024-12-06T00:45:49,274 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/ns/0c716a72bccc4e1e9b3bcc9b9a862c39, entries=2, sequenceid=13, filesize=5.0 K 2024-12-06T00:45:49,275 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/.tmp/table/216f65a7b5ca4082b0a79c2703895415 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/table/216f65a7b5ca4082b0a79c2703895415 2024-12-06T00:45:49,277 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/table/216f65a7b5ca4082b0a79c2703895415, delay=959 ms 2024-12-06T00:45:49,278 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/table/216f65a7b5ca4082b0a79c2703895415, entries=2, sequenceid=13, filesize=5.3 K 2024-12-06T00:45:49,279 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=13, compaction requested=false 2024-12-06T00:45:49,280 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/info/bda36d91dd454942adebf065334a7caa 2024-12-06T00:45:49,280 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/ns/0c716a72bccc4e1e9b3bcc9b9a862c39 2024-12-06T00:45:49,281 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/table/216f65a7b5ca4082b0a79c2703895415 2024-12-06T00:45:49,286 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-06T00:45:49,287 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T00:45:49,287 INFO [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T00:45:49,287 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733445949185Running coprocessor pre-close hooks at 1733445949185Disabling compacts and flushes for region at 1733445949185Disabling writes for close at 1733445949186 (+1 ms)Obtaining lock to block concurrent updates at 1733445949187 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733445949187Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733445949187Flushing stores of hbase:meta,,1.1588230740 at 1733445949188 (+1 ms)Flushing 1588230740/info: creating writer at 1733445949188Flushing 1588230740/info: appending metadata at 1733445949215 (+27 ms)Flushing 1588230740/info: closing flushed file at 1733445949215Flushing 1588230740/ns: creating writer at 1733445949218 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733445949237 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1733445949237Flushing 1588230740/table: creating writer at 1733445949240 (+3 ms)Flushing 1588230740/table: appending metadata at 1733445949263 (+23 ms)Flushing 1588230740/table: closing flushed file at 1733445949263Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b6d56f9: reopening flushed file at 1733445949267 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5089932b: reopening flushed file at 1733445949271 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@451c1de2: reopening flushed file at 1733445949275 (+4 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=13, compaction requested=false at 1733445949279 (+4 ms)Writing region close event to WAL at 1733445949281 (+2 ms)Running coprocessor post-close hooks at 1733445949287 (+6 ms)Closed at 1733445949287 2024-12-06T00:45:49,287 DEBUG [RS_CLOSE_META-regionserver/f7c814cf3c7e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T00:45:49,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39181-0x10064c9c0600003, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:49,293 INFO [RS:2;f7c814cf3c7e:39181 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T00:45:49,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39181-0x10064c9c0600003, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:49,293 INFO [RS:2;f7c814cf3c7e:39181 {}] regionserver.HRegionServer(1031): Exiting; stopping=f7c814cf3c7e,39181,1733445948560; zookeeper connection closed. 2024-12-06T00:45:49,293 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50cd93c7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50cd93c7 2024-12-06T00:45:49,386 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(976): stopping server f7c814cf3c7e,40783,1733445945313; all regions closed. 2024-12-06T00:45:49,388 DEBUG [RS:1;f7c814cf3c7e:40783 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs 2024-12-06T00:45:49,388 INFO [RS:1;f7c814cf3c7e:40783 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7c814cf3c7e%2C40783%2C1733445945313.meta:.meta(num 1733445946201) 2024-12-06T00:45:49,389 DEBUG [RS:1;f7c814cf3c7e:40783 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/oldWALs 2024-12-06T00:45:49,389 INFO [RS:1;f7c814cf3c7e:40783 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7c814cf3c7e%2C40783%2C1733445945313:(num 1733445945833) 2024-12-06T00:45:49,389 DEBUG [RS:1;f7c814cf3c7e:40783 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T00:45:49,389 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T00:45:49,389 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T00:45:49,390 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.ChoreService(370): Chore service for: regionserver/f7c814cf3c7e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T00:45:49,390 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T00:45:49,390 INFO [regionserver/f7c814cf3c7e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T00:45:49,390 INFO [RS:1;f7c814cf3c7e:40783 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40783 2024-12-06T00:45:49,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f7c814cf3c7e,40783,1733445945313 2024-12-06T00:45:49,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T00:45:49,392 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T00:45:49,394 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f7c814cf3c7e,40783,1733445945313] 2024-12-06T00:45:49,395 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f7c814cf3c7e,40783,1733445945313 already deleted, retry=false 2024-12-06T00:45:49,395 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f7c814cf3c7e,40783,1733445945313 expired; onlineServers=0 2024-12-06T00:45:49,395 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f7c814cf3c7e,43179,1733445945227' ***** 2024-12-06T00:45:49,395 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T00:45:49,396 INFO [M:0;f7c814cf3c7e:43179 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T00:45:49,396 INFO [M:0;f7c814cf3c7e:43179 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T00:45:49,396 DEBUG [M:0;f7c814cf3c7e:43179 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T00:45:49,396 DEBUG [M:0;f7c814cf3c7e:43179 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T00:45:49,396 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T00:45:49,396 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.large.0-1733445945569 {}] cleaner.HFileCleaner(306): Exit Thread[master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.large.0-1733445945569,5,FailOnTimeoutGroup] 2024-12-06T00:45:49,396 DEBUG [master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.small.0-1733445945572 {}] cleaner.HFileCleaner(306): Exit Thread[master/f7c814cf3c7e:0:becomeActiveMaster-HFileCleaner.small.0-1733445945572,5,FailOnTimeoutGroup] 2024-12-06T00:45:49,396 INFO [M:0;f7c814cf3c7e:43179 {}] hbase.ChoreService(370): Chore service for: master/f7c814cf3c7e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T00:45:49,396 INFO [M:0;f7c814cf3c7e:43179 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T00:45:49,396 DEBUG [M:0;f7c814cf3c7e:43179 {}] master.HMaster(1795): Stopping service threads 2024-12-06T00:45:49,396 INFO [M:0;f7c814cf3c7e:43179 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T00:45:49,397 INFO [M:0;f7c814cf3c7e:43179 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T00:45:49,397 INFO [M:0;f7c814cf3c7e:43179 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T00:45:49,397 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T00:45:49,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T00:45:49,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T00:45:49,398 DEBUG [M:0;f7c814cf3c7e:43179 {}] zookeeper.ZKUtil(347): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T00:45:49,398 WARN [M:0;f7c814cf3c7e:43179 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T00:45:49,398 INFO [M:0;f7c814cf3c7e:43179 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/.lastflushedseqids 2024-12-06T00:45:49,405 INFO [M:0;f7c814cf3c7e:43179 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T00:45:49,405 INFO [M:0;f7c814cf3c7e:43179 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T00:45:49,406 DEBUG [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T00:45:49,406 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:49,406 DEBUG [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:49,406 DEBUG [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T00:45:49,406 DEBUG [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:49,406 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-12-06T00:45:49,416 DEBUG [M:0;f7c814cf3c7e:43179 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c109c2df62764ce68e112ab8458b8407 2024-12-06T00:45:49,429 DEBUG [M:0;f7c814cf3c7e:43179 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c109c2df62764ce68e112ab8458b8407 is 82, key is hbase:meta,,1/info:regioninfo/1733445946240/Put/seqid=0 2024-12-06T00:45:49,430 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c109c2df62764ce68e112ab8458b8407 2024-12-06T00:45:49,438 DEBUG [M:0;f7c814cf3c7e:43179 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 81246c3a8a134b709529957f1801708e 2024-12-06T00:45:49,452 DEBUG [M:0;f7c814cf3c7e:43179 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/81246c3a8a134b709529957f1801708e is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733445946743/Put/seqid=0 2024-12-06T00:45:49,453 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/81246c3a8a134b709529957f1801708e 2024-12-06T00:45:49,461 DEBUG [M:0;f7c814cf3c7e:43179 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b6244ea283ce43d1b8fb7af45e8f93d3 2024-12-06T00:45:49,475 DEBUG [M:0;f7c814cf3c7e:43179 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b6244ea283ce43d1b8fb7af45e8f93d3 is 69, key is f7c814cf3c7e,39181,1733445948560/rs:state/1733445948603/Put/seqid=0 2024-12-06T00:45:49,476 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b6244ea283ce43d1b8fb7af45e8f93d3 2024-12-06T00:45:49,478 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b6244ea283ce43d1b8fb7af45e8f93d3 2024-12-06T00:45:49,478 DEBUG [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c109c2df62764ce68e112ab8458b8407 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c109c2df62764ce68e112ab8458b8407 2024-12-06T00:45:49,480 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c109c2df62764ce68e112ab8458b8407, entries=8, sequenceid=83, filesize=5.5 K 2024-12-06T00:45:49,481 DEBUG [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/81246c3a8a134b709529957f1801708e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/81246c3a8a134b709529957f1801708e 2024-12-06T00:45:49,482 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/81246c3a8a134b709529957f1801708e, entries=9, sequenceid=83, filesize=6.4 K 2024-12-06T00:45:49,482 DEBUG [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b6244ea283ce43d1b8fb7af45e8f93d3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b6244ea283ce43d1b8fb7af45e8f93d3 2024-12-06T00:45:49,484 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b6244ea283ce43d1b8fb7af45e8f93d3 2024-12-06T00:45:49,484 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4b4e8c5-52f7-cbe1-32c5-ad831fe4a9ee/685fea38-8d23-f514-7c75-f3dd0ae837c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b6244ea283ce43d1b8fb7af45e8f93d3, entries=3, sequenceid=83, filesize=5.3 K 2024-12-06T00:45:49,485 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=83, compaction requested=false 2024-12-06T00:45:49,486 INFO [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T00:45:49,486 DEBUG [M:0;f7c814cf3c7e:43179 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733445949406Disabling compacts and flushes for region at 1733445949406Disabling writes for close at 1733445949406Obtaining lock to block concurrent updates at 1733445949406Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733445949406Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1733445949406Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733445949407 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733445949407Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733445949429 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733445949429Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733445949432 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733445949452 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733445949452Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733445949454 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733445949474 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733445949474Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18b09da7: reopening flushed file at 1733445949478 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@179b5d1a: reopening flushed file at 1733445949480 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b567a08: reopening flushed file at 1733445949482 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=83, compaction requested=false at 1733445949485 (+3 ms)Writing region close event to WAL at 1733445949486 (+1 ms)Closed at 1733445949486 2024-12-06T00:45:49,488 INFO [M:0;f7c814cf3c7e:43179 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T00:45:49,488 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T00:45:49,488 INFO [M:0;f7c814cf3c7e:43179 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43179 2024-12-06T00:45:49,488 INFO [M:0;f7c814cf3c7e:43179 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T00:45:49,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:49,494 INFO [RS:1;f7c814cf3c7e:40783 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T00:45:49,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40783-0x10064c9c0600002, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:49,494 INFO [RS:1;f7c814cf3c7e:40783 {}] regionserver.HRegionServer(1031): Exiting; stopping=f7c814cf3c7e,40783,1733445945313; zookeeper connection closed. 2024-12-06T00:45:49,494 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4d7c940f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4d7c940f 2024-12-06T00:45:49,495 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-06T00:45:49,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:49,590 INFO [M:0;f7c814cf3c7e:43179 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T00:45:49,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43179-0x10064c9c0600000, quorum=127.0.0.1:49718, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T00:45:49,593 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T00:45:49,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-06T00:45:49,594 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T00:45:49,601 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1064 (was 1148), ProcessCount=11 (was 11), AvailableMemoryMB=11556 (was 11618)