2024-11-26 09:45:17,254 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-26 09:45:17,265 main DEBUG Took 0.009701 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 09:45:17,266 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 09:45:17,266 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 09:45:17,267 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 09:45:17,268 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,275 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 09:45:17,295 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,297 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,298 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,299 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,300 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,301 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,302 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,303 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,304 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,304 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,305 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,305 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,306 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,307 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,307 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,308 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,309 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,310 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,311 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,311 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:17,312 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,312 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 09:45:17,314 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:17,315 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 09:45:17,317 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 09:45:17,317 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 09:45:17,319 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 09:45:17,319 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 09:45:17,328 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 09:45:17,330 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 09:45:17,332 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 09:45:17,332 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 09:45:17,332 main DEBUG createAppenders(={Console}) 2024-11-26 09:45:17,333 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-26 09:45:17,333 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-26 09:45:17,334 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-26 09:45:17,334 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 09:45:17,334 main DEBUG OutputStream closed 2024-11-26 09:45:17,335 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 09:45:17,335 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 09:45:17,335 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-26 09:45:17,400 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 09:45:17,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 09:45:17,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 09:45:17,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 09:45:17,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 09:45:17,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 09:45:17,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 09:45:17,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 09:45:17,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 09:45:17,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 09:45:17,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 09:45:17,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 09:45:17,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 09:45:17,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 09:45:17,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 09:45:17,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 09:45:17,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 09:45:17,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 09:45:17,411 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 09:45:17,412 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-26 09:45:17,412 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 09:45:17,413 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-26T09:45:17,426 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-26 09:45:17,429 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 09:45:17,429 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T09:45:17,705 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786 2024-11-26T09:45:17,729 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1396, ProcessCount=11, AvailableMemoryMB=4579 2024-11-26T09:45:17,828 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T09:45:17,928 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:17,944 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/cluster_75c97f12-fb46-caa8-6c62-04f129515ce6, deleteOnExit=true 2024-11-26T09:45:18,034 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/cluster_75c97f12-fb46-caa8-6c62-04f129515ce6/zookeeper_0, clientPort=61297, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/cluster_75c97f12-fb46-caa8-6c62-04f129515ce6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/cluster_75c97f12-fb46-caa8-6c62-04f129515ce6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T09:45:18,048 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61297 2024-11-26T09:45:18,048 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:18,135 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 with version=8 2024-11-26T09:45:18,136 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/hbase-staging 2024-11-26T09:45:18,239 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T09:45:18,511 INFO [Time-limited test {}] client.ConnectionUtils(128): master/24c191b0ea9d:0 server-side Connection retries=45 2024-11-26T09:45:18,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:18,524 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:18,532 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T09:45:18,532 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:18,532 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T09:45:18,670 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T09:45:18,760 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T09:45:18,797 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T09:45:18,808 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T09:45:18,812 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:18,854 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14511 (auto-detected) 2024-11-26T09:45:18,855 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T09:45:18,875 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36883 2024-11-26T09:45:18,898 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36883 connecting to ZooKeeper ensemble=127.0.0.1:61297 2024-11-26T09:45:18,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:368830x0, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T09:45:18,936 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36883-0x1003338209d0000 connected 2024-11-26T09:45:18,965 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:18,966 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:18,973 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:18,978 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73, hbase.cluster.distributed=false 2024-11-26T09:45:19,006 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T09:45:19,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36883 2024-11-26T09:45:19,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36883 2024-11-26T09:45:19,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36883 2024-11-26T09:45:19,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36883 2024-11-26T09:45:19,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36883 2024-11-26T09:45:19,102 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/24c191b0ea9d:0 server-side Connection retries=45 2024-11-26T09:45:19,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:19,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:19,105 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T09:45:19,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:19,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T09:45:19,108 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T09:45:19,111 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:19,112 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43075 2024-11-26T09:45:19,114 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43075 connecting to ZooKeeper ensemble=127.0.0.1:61297 2024-11-26T09:45:19,114 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:19,115 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:19,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430750x0, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T09:45:19,120 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43075-0x1003338209d0001 connected 2024-11-26T09:45:19,120 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:19,126 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T09:45:19,138 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/bucket.cache 2024-11-26T09:45:19,143 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T09:45:19,145 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T09:45:19,146 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T09:45:19,148 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T09:45:19,148 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T09:45:19,154 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T09:45:19,159 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T09:45:19,160 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T09:45:19,161 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/bucket.persistence is complete. 2024-11-26T09:45:19,162 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T09:45:19,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T09:45:19,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43075 2024-11-26T09:45:19,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43075 2024-11-26T09:45:19,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43075 2024-11-26T09:45:19,175 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43075 2024-11-26T09:45:19,175 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43075 2024-11-26T09:45:19,195 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/24c191b0ea9d:0 server-side Connection retries=45 2024-11-26T09:45:19,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:19,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:19,195 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T09:45:19,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:19,196 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T09:45:19,196 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T09:45:19,196 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:19,197 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41767 2024-11-26T09:45:19,199 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41767 connecting to ZooKeeper ensemble=127.0.0.1:61297 2024-11-26T09:45:19,199 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:19,200 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:19,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417670x0, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T09:45:19,205 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41767-0x1003338209d0002 connected 2024-11-26T09:45:19,206 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:19,206 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T09:45:19,208 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/bucket.cache 2024-11-26T09:45:19,208 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T09:45:19,208 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T09:45:19,208 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T09:45:19,210 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T09:45:19,210 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T09:45:19,210 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T09:45:19,211 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T09:45:19,211 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T09:45:19,211 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/bucket.persistence is complete. 2024-11-26T09:45:19,212 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T09:45:19,215 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T09:45:19,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41767 2024-11-26T09:45:19,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41767 2024-11-26T09:45:19,219 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41767 2024-11-26T09:45:19,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41767 2024-11-26T09:45:19,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41767 2024-11-26T09:45:19,239 DEBUG [M:0;24c191b0ea9d:36883 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;24c191b0ea9d:36883 2024-11-26T09:45:19,240 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/24c191b0ea9d,36883,1732614318310 2024-11-26T09:45:19,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:19,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:19,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:19,253 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/24c191b0ea9d,36883,1732614318310 2024-11-26T09:45:19,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T09:45:19,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T09:45:19,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:19,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:19,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:19,278 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T09:45:19,279 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/24c191b0ea9d,36883,1732614318310 from backup master directory 2024-11-26T09:45:19,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:19,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/24c191b0ea9d,36883,1732614318310 2024-11-26T09:45:19,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:19,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:19,283 WARN [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T09:45:19,283 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=24c191b0ea9d,36883,1732614318310 2024-11-26T09:45:19,285 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T09:45:19,287 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T09:45:19,295 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/hbase.id] with ID: 2941950a-5bdd-44aa-9728-5a972b63b0d0 2024-11-26T09:45:19,295 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/.tmp/hbase.id 2024-11-26T09:45:19,302 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/hbase.id] 2024-11-26T09:45:19,345 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:19,350 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T09:45:19,370 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-26T09:45:19,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:19,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:19,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:19,404 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T09:45:19,408 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T09:45:19,457 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:45:19,463 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T09:45:19,528 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store 2024-11-26T09:45:19,552 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T09:45:19,557 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:19,559 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T09:45:19,560 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:19,560 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:19,562 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T09:45:19,563 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:19,563 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:19,564 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732614319559Disabling compacts and flushes for region at 1732614319559Disabling writes for close at 1732614319562 (+3 ms)Writing region close event to WAL at 1732614319563 (+1 ms)Closed at 1732614319563 2024-11-26T09:45:19,565 WARN [master/24c191b0ea9d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/.initializing 2024-11-26T09:45:19,565 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/WALs/24c191b0ea9d,36883,1732614318310 2024-11-26T09:45:19,579 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T09:45:19,594 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=24c191b0ea9d%2C36883%2C1732614318310, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/WALs/24c191b0ea9d,36883,1732614318310, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/oldWALs, maxLogs=10 2024-11-26T09:45:19,635 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/WALs/24c191b0ea9d,36883,1732614318310/24c191b0ea9d%2C36883%2C1732614318310.1732614319600 2024-11-26T09:45:19,636 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T09:45:19,637 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T09:45:19,638 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:19,641 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,643 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T09:45:19,717 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:19,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:19,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T09:45:19,726 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:19,727 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:19,727 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,733 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T09:45:19,733 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:19,734 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:19,734 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T09:45:19,740 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:19,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:19,741 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,744 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,745 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,748 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,749 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,751 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T09:45:19,754 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:19,766 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T09:45:19,767 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62655330, jitterRate=-0.06636282801628113}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T09:45:19,776 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732614319656Initializing all the Stores at 1732614319657 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614319657Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614319658 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614319659 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614319659Cleaning up temporary data from old regions at 1732614319749 (+90 ms)Region opened successfully at 1732614319776 (+27 ms) 2024-11-26T09:45:19,778 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T09:45:19,823 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1af41e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=24c191b0ea9d/172.17.0.2:0 2024-11-26T09:45:19,871 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T09:45:19,883 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T09:45:19,883 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T09:45:19,887 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T09:45:19,887 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T09:45:19,893 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-26T09:45:19,894 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T09:45:19,919 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T09:45:19,929 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T09:45:19,931 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T09:45:19,933 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T09:45:19,935 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T09:45:19,937 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T09:45:19,939 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T09:45:19,943 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T09:45:19,944 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T09:45:19,946 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T09:45:19,948 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T09:45:19,968 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T09:45:19,970 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T09:45:19,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:19,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:19,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:19,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:19,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:19,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:19,988 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=24c191b0ea9d,36883,1732614318310, sessionid=0x1003338209d0000, setting cluster-up flag (Was=false) 2024-11-26T09:45:20,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:20,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:20,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:20,009 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T09:45:20,012 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=24c191b0ea9d,36883,1732614318310 2024-11-26T09:45:20,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:20,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:20,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:20,022 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T09:45:20,024 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=24c191b0ea9d,36883,1732614318310 2024-11-26T09:45:20,029 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T09:45:20,106 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T09:45:20,117 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T09:45:20,124 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T09:45:20,126 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(746): ClusterId : 2941950a-5bdd-44aa-9728-5a972b63b0d0 2024-11-26T09:45:20,126 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(746): ClusterId : 2941950a-5bdd-44aa-9728-5a972b63b0d0 2024-11-26T09:45:20,129 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T09:45:20,129 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T09:45:20,135 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T09:45:20,135 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T09:45:20,135 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T09:45:20,135 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T09:45:20,133 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 24c191b0ea9d,36883,1732614318310 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T09:45:20,138 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T09:45:20,138 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T09:45:20,139 DEBUG [RS:1;24c191b0ea9d:41767 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28bb6e04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=24c191b0ea9d/172.17.0.2:0 2024-11-26T09:45:20,139 DEBUG [RS:0;24c191b0ea9d:43075 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4a7ba5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=24c191b0ea9d/172.17.0.2:0 2024-11-26T09:45:20,142 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/24c191b0ea9d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T09:45:20,143 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/24c191b0ea9d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T09:45:20,143 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/24c191b0ea9d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T09:45:20,143 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/24c191b0ea9d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T09:45:20,143 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/24c191b0ea9d:0, corePoolSize=10, maxPoolSize=10 2024-11-26T09:45:20,143 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,144 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/24c191b0ea9d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T09:45:20,144 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,148 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732614350148 2024-11-26T09:45:20,149 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T09:45:20,149 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T09:45:20,150 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T09:45:20,151 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T09:45:20,154 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:20,155 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T09:45:20,155 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T09:45:20,156 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T09:45:20,156 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T09:45:20,156 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T09:45:20,157 DEBUG [RS:0;24c191b0ea9d:43075 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;24c191b0ea9d:43075 2024-11-26T09:45:20,159 DEBUG [RS:1;24c191b0ea9d:41767 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;24c191b0ea9d:41767 2024-11-26T09:45:20,157 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,161 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T09:45:20,161 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T09:45:20,161 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T09:45:20,162 DEBUG [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T09:45:20,162 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T09:45:20,162 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T09:45:20,162 DEBUG [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T09:45:20,163 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T09:45:20,164 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T09:45:20,166 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(2659): reportForDuty to master=24c191b0ea9d,36883,1732614318310 with port=41767, startcode=1732614319194 2024-11-26T09:45:20,166 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(2659): reportForDuty to master=24c191b0ea9d,36883,1732614318310 with port=43075, startcode=1732614319063 2024-11-26T09:45:20,167 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T09:45:20,167 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T09:45:20,169 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.large.0-1732614320169,5,FailOnTimeoutGroup] 2024-11-26T09:45:20,170 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.small.0-1732614320169,5,FailOnTimeoutGroup] 2024-11-26T09:45:20,170 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,170 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T09:45:20,172 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,172 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,182 DEBUG [RS:1;24c191b0ea9d:41767 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T09:45:20,182 DEBUG [RS:0;24c191b0ea9d:43075 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T09:45:20,182 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T09:45:20,182 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 2024-11-26T09:45:20,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:20,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T09:45:20,198 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T09:45:20,198 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:20,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:20,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T09:45:20,203 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T09:45:20,203 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:20,204 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:20,204 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T09:45:20,211 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T09:45:20,211 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:20,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:20,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T09:45:20,217 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T09:45:20,218 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:20,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:20,218 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T09:45:20,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740 2024-11-26T09:45:20,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740 2024-11-26T09:45:20,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T09:45:20,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T09:45:20,220 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T09:45:20,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T09:45:20,231 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T09:45:20,236 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65622415, jitterRate=-0.022149816155433655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T09:45:20,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732614320192Initializing all the Stores at 1732614320192Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614320192Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614320193 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614320193Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614320193Cleaning up temporary data from old regions at 1732614320220 (+27 ms)Region opened successfully at 1732614320239 (+19 ms) 2024-11-26T09:45:20,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T09:45:20,240 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T09:45:20,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T09:45:20,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T09:45:20,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T09:45:20,242 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T09:45:20,242 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732614320239Disabling compacts and flushes for region at 1732614320239Disabling writes for close at 1732614320240 (+1 ms)Writing region close event to WAL at 1732614320241 (+1 ms)Closed at 1732614320242 (+1 ms) 2024-11-26T09:45:20,245 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T09:45:20,245 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T09:45:20,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T09:45:20,264 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46381, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T09:45:20,264 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58127, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T09:45:20,264 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T09:45:20,266 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T09:45:20,270 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36883 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:20,273 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36883 {}] master.ServerManager(517): Registering regionserver=24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:20,283 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36883 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:20,283 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36883 {}] master.ServerManager(517): Registering regionserver=24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:20,287 DEBUG [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 2024-11-26T09:45:20,287 DEBUG [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 2024-11-26T09:45:20,287 DEBUG [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T09:45:20,287 DEBUG [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T09:45:20,287 DEBUG [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T09:45:20,287 DEBUG [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T09:45:20,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T09:45:20,295 DEBUG [RS:1;24c191b0ea9d:41767 {}] zookeeper.ZKUtil(111): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:20,295 DEBUG [RS:0;24c191b0ea9d:43075 {}] zookeeper.ZKUtil(111): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:20,295 WARN [RS:0;24c191b0ea9d:43075 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T09:45:20,295 WARN [RS:1;24c191b0ea9d:41767 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T09:45:20,295 INFO [RS:0;24c191b0ea9d:43075 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T09:45:20,295 INFO [RS:1;24c191b0ea9d:41767 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T09:45:20,296 DEBUG [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:20,296 DEBUG [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:20,297 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [24c191b0ea9d,43075,1732614319063] 2024-11-26T09:45:20,297 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [24c191b0ea9d,41767,1732614319194] 2024-11-26T09:45:20,323 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T09:45:20,323 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T09:45:20,340 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T09:45:20,340 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T09:45:20,346 INFO [RS:0;24c191b0ea9d:43075 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T09:45:20,346 INFO [RS:1;24c191b0ea9d:41767 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T09:45:20,346 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,346 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,348 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T09:45:20,348 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T09:45:20,353 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T09:45:20,353 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T09:45:20,355 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,355 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,355 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,355 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,355 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,355 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,356 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,356 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,356 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,356 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,356 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,356 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/24c191b0ea9d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T09:45:20,356 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,356 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,356 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/24c191b0ea9d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T09:45:20,357 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,357 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,357 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,357 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,357 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,357 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,357 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,357 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,357 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/24c191b0ea9d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T09:45:20,357 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,358 DEBUG [RS:1;24c191b0ea9d:41767 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/24c191b0ea9d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T09:45:20,358 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,358 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:20,358 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/24c191b0ea9d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T09:45:20,358 DEBUG [RS:0;24c191b0ea9d:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/24c191b0ea9d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T09:45:20,360 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,360 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,360 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,361 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,361 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,361 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,361 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,361 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,41767,1732614319194-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T09:45:20,361 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,361 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,361 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,362 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,43075,1732614319063-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T09:45:20,384 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T09:45:20,384 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T09:45:20,386 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,41767,1732614319194-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,386 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,43075,1732614319063-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,387 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,387 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,387 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.Replication(171): 24c191b0ea9d,41767,1732614319194 started 2024-11-26T09:45:20,387 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.Replication(171): 24c191b0ea9d,43075,1732614319063 started 2024-11-26T09:45:20,406 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,407 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(1482): Serving as 24c191b0ea9d,41767,1732614319194, RpcServer on 24c191b0ea9d/172.17.0.2:41767, sessionid=0x1003338209d0002 2024-11-26T09:45:20,408 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T09:45:20,408 DEBUG [RS:1;24c191b0ea9d:41767 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:20,408 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '24c191b0ea9d,41767,1732614319194' 2024-11-26T09:45:20,408 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T09:45:20,409 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T09:45:20,410 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T09:45:20,410 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T09:45:20,410 DEBUG [RS:1;24c191b0ea9d:41767 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:20,410 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '24c191b0ea9d,41767,1732614319194' 2024-11-26T09:45:20,410 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T09:45:20,411 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T09:45:20,412 DEBUG [RS:1;24c191b0ea9d:41767 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T09:45:20,412 INFO [RS:1;24c191b0ea9d:41767 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T09:45:20,412 INFO [RS:1;24c191b0ea9d:41767 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T09:45:20,413 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:20,413 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(1482): Serving as 24c191b0ea9d,43075,1732614319063, RpcServer on 24c191b0ea9d/172.17.0.2:43075, sessionid=0x1003338209d0001 2024-11-26T09:45:20,413 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T09:45:20,414 DEBUG [RS:0;24c191b0ea9d:43075 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:20,414 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '24c191b0ea9d,43075,1732614319063' 2024-11-26T09:45:20,414 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T09:45:20,414 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T09:45:20,415 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T09:45:20,415 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T09:45:20,415 DEBUG [RS:0;24c191b0ea9d:43075 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:20,415 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '24c191b0ea9d,43075,1732614319063' 2024-11-26T09:45:20,415 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T09:45:20,416 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T09:45:20,417 DEBUG [RS:0;24c191b0ea9d:43075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T09:45:20,417 INFO [RS:0;24c191b0ea9d:43075 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T09:45:20,417 INFO [RS:0;24c191b0ea9d:43075 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T09:45:20,417 WARN [24c191b0ea9d:36883 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T09:45:20,518 INFO [RS:0;24c191b0ea9d:43075 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T09:45:20,518 INFO [RS:1;24c191b0ea9d:41767 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T09:45:20,519 INFO [RS:0;24c191b0ea9d:43075 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=24c191b0ea9d%2C43075%2C1732614319063, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,43075,1732614319063, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs, maxLogs=55 2024-11-26T09:45:20,519 INFO [RS:1;24c191b0ea9d:41767 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=24c191b0ea9d%2C41767%2C1732614319194, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,41767,1732614319194, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs, maxLogs=55 2024-11-26T09:45:20,543 INFO [RS:0;24c191b0ea9d:43075 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,43075,1732614319063/24c191b0ea9d%2C43075%2C1732614319063.1732614320523 2024-11-26T09:45:20,544 INFO [RS:1;24c191b0ea9d:41767 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,41767,1732614319194/24c191b0ea9d%2C41767%2C1732614319194.1732614320523 2024-11-26T09:45:20,544 DEBUG [RS:0;24c191b0ea9d:43075 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T09:45:20,544 DEBUG [RS:1;24c191b0ea9d:41767 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T09:45:20,670 DEBUG [24c191b0ea9d:36883 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T09:45:20,682 DEBUG [24c191b0ea9d:36883 {}] balancer.BalancerClusterState(204): Hosts are {24c191b0ea9d=0} racks are {/default-rack=0} 2024-11-26T09:45:20,685 DEBUG [24c191b0ea9d:36883 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T09:45:20,685 DEBUG [24c191b0ea9d:36883 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T09:45:20,686 DEBUG [24c191b0ea9d:36883 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T09:45:20,686 DEBUG [24c191b0ea9d:36883 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T09:45:20,686 INFO [24c191b0ea9d:36883 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T09:45:20,686 INFO [24c191b0ea9d:36883 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T09:45:20,686 DEBUG [24c191b0ea9d:36883 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T09:45:20,693 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:20,699 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 24c191b0ea9d,43075,1732614319063, state=OPENING 2024-11-26T09:45:20,704 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T09:45:20,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:20,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:20,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:20,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:20,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:20,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:20,708 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T09:45:20,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=24c191b0ea9d,43075,1732614319063}] 2024-11-26T09:45:20,884 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T09:45:20,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52547, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T09:45:20,896 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T09:45:20,897 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T09:45:20,897 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T09:45:20,898 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=24c191b0ea9d%2C43075%2C1732614319063.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,43075,1732614319063, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs, maxLogs=55 2024-11-26T09:45:20,919 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,43075,1732614319063/24c191b0ea9d%2C43075%2C1732614319063.meta.1732614320900.meta 2024-11-26T09:45:20,920 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T09:45:20,920 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T09:45:20,922 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T09:45:20,925 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T09:45:20,929 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T09:45:20,933 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T09:45:20,934 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:20,934 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T09:45:20,934 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T09:45:20,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T09:45:20,937 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T09:45:20,937 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:20,937 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:20,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T09:45:20,938 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T09:45:20,938 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:20,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:20,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T09:45:20,940 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T09:45:20,940 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:20,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:20,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T09:45:20,941 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T09:45:20,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:20,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:20,941 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T09:45:20,942 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740 2024-11-26T09:45:20,943 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740 2024-11-26T09:45:20,943 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T09:45:20,943 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T09:45:20,944 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T09:45:20,945 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T09:45:20,945 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74372209, jitterRate=0.1082322746515274}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T09:45:20,946 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T09:45:20,947 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732614320935Writing region info on filesystem at 1732614320935Initializing all the Stores at 1732614320935Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614320935Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614320936 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614320936Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614320936Cleaning up temporary data from old regions at 1732614320944 (+8 ms)Running coprocessor post-open hooks at 1732614320946 (+2 ms)Region opened successfully at 1732614320947 (+1 ms) 2024-11-26T09:45:20,953 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732614320875 2024-11-26T09:45:20,964 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T09:45:20,964 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T09:45:20,965 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:20,966 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 24c191b0ea9d,43075,1732614319063, state=OPEN 2024-11-26T09:45:20,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:20,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:20,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:20,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:20,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:20,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:20,969 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:20,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T09:45:20,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=24c191b0ea9d,43075,1732614319063 in 259 msec 2024-11-26T09:45:20,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T09:45:20,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 720 msec 2024-11-26T09:45:20,979 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T09:45:20,979 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T09:45:21,001 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T09:45:21,003 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=24c191b0ea9d,43075,1732614319063, seqNum=-1] 2024-11-26T09:45:21,022 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T09:45:21,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46643, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T09:45:21,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 982 msec 2024-11-26T09:45:21,043 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732614321042, completionTime=-1 2024-11-26T09:45:21,045 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-26T09:45:21,046 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T09:45:21,072 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T09:45:21,072 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732614381072 2024-11-26T09:45:21,072 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732614441072 2024-11-26T09:45:21,072 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-26T09:45:21,075 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T09:45:21,081 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,36883,1732614318310-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:21,081 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,36883,1732614318310-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:21,081 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,36883,1732614318310-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:21,083 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-24c191b0ea9d:36883, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:21,083 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:21,084 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:21,090 DEBUG [master/24c191b0ea9d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T09:45:21,115 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.832sec 2024-11-26T09:45:21,117 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T09:45:21,118 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T09:45:21,119 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T09:45:21,120 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T09:45:21,120 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T09:45:21,121 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,36883,1732614318310-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T09:45:21,121 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,36883,1732614318310-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T09:45:21,122 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T09:45:21,123 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T09:45:21,123 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,36883,1732614318310-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:21,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:21,138 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T09:45:21,138 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T09:45:21,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 24c191b0ea9d,36883,-1 for getting cluster id 2024-11-26T09:45:21,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T09:45:21,171 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2941950a-5bdd-44aa-9728-5a972b63b0d0' 2024-11-26T09:45:21,174 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T09:45:21,174 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2941950a-5bdd-44aa-9728-5a972b63b0d0" 2024-11-26T09:45:21,176 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@616a61f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:21,177 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [24c191b0ea9d,36883,-1] 2024-11-26T09:45:21,179 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T09:45:21,181 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:21,182 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T09:45:21,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:21,186 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T09:45:21,193 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=24c191b0ea9d,43075,1732614319063, seqNum=-1] 2024-11-26T09:45:21,193 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T09:45:21,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T09:45:21,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=24c191b0ea9d,36883,1732614318310 2024-11-26T09:45:21,219 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T09:45:21,224 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 24c191b0ea9d,36883,1732614318310 2024-11-26T09:45:21,226 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26ed5dcb 2024-11-26T09:45:21,227 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T09:45:21,230 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T09:45:21,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732614321215', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T09:45:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732614321215 2024-11-26T09:45:21,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732614321215 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T09:45:21,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732614321215" procId is: 4 2024-11-26T09:45:21,250 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:21,251 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732614321215 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T09:45:21,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T09:45:21,265 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732614321215-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 95bf97a536a72d11a71086d876c972e2, NAME => 'testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732614321215', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 2024-11-26T09:45:21,272 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732614321215-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:21,272 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732614321215-pool-0 {}] regionserver.HRegion(1722): Closing 95bf97a536a72d11a71086d876c972e2, disabling compactions & flushes 2024-11-26T09:45:21,272 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732614321215-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:21,272 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732614321215-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:21,272 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732614321215-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. after waiting 0 ms 2024-11-26T09:45:21,272 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732614321215-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:21,272 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732614321215-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:21,273 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732614321215-pool-0 {}] regionserver.HRegion(1676): Region close journal for 95bf97a536a72d11a71086d876c972e2: Waiting for close lock at 1732614321272Disabling compacts and flushes for region at 1732614321272Disabling writes for close at 1732614321272Writing region close event to WAL at 1732614321272Closed at 1732614321272 2024-11-26T09:45:21,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732614321215 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T09:45:21,279 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732614321274"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732614321274"}]},"ts":"1732614321274"} 2024-11-26T09:45:21,286 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T09:45:21,287 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732614321215 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T09:45:21,290 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732614321215","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732614321287"}]},"ts":"1732614321287"} 2024-11-26T09:45:21,294 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732614321215, state=ENABLING in hbase:meta 2024-11-26T09:45:21,295 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {24c191b0ea9d=0} racks are {/default-rack=0} 2024-11-26T09:45:21,296 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T09:45:21,296 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T09:45:21,296 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T09:45:21,296 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T09:45:21,296 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T09:45:21,296 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T09:45:21,296 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T09:45:21,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, ASSIGN}] 2024-11-26T09:45:21,299 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, ASSIGN 2024-11-26T09:45:21,300 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, ASSIGN; state=OFFLINE, location=24c191b0ea9d,43075,1732614319063; forceNewPlan=false, retain=false 2024-11-26T09:45:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T09:45:21,452 INFO [24c191b0ea9d:36883 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T09:45:21,453 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=95bf97a536a72d11a71086d876c972e2, regionState=OPENING, regionLocation=24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:21,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, ASSIGN because future has completed 2024-11-26T09:45:21,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95bf97a536a72d11a71086d876c972e2, server=24c191b0ea9d,43075,1732614319063}] 2024-11-26T09:45:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T09:45:21,613 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:21,614 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 95bf97a536a72d11a71086d876c972e2, NAME => 'testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2.', STARTKEY => '', ENDKEY => ''} 2024-11-26T09:45:21,614 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732614321215 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,614 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:21,614 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,615 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,615 INFO [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,620 INFO [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95bf97a536a72d11a71086d876c972e2 columnFamilyName family 2024-11-26T09:45:21,620 DEBUG [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:21,620 INFO [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] regionserver.HStore(327): Store=95bf97a536a72d11a71086d876c972e2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:21,621 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,621 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,621 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,622 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,622 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,622 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,629 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T09:45:21,630 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 95bf97a536a72d11a71086d876c972e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74970989, jitterRate=0.1171547919511795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T09:45:21,630 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:21,631 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 95bf97a536a72d11a71086d876c972e2: Running coprocessor pre-open hook at 1732614321615Writing region info on filesystem at 1732614321615Initializing all the Stores at 1732614321615Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614321615Cleaning up temporary data from old regions at 1732614321622 (+7 ms)Running coprocessor post-open hooks at 1732614321630 (+8 ms)Region opened successfully at 1732614321631 (+1 ms) 2024-11-26T09:45:21,632 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2., pid=6, masterSystemTime=1732614321610 2024-11-26T09:45:21,635 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:21,635 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:21,636 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=95bf97a536a72d11a71086d876c972e2, regionState=OPEN, openSeqNum=2, regionLocation=24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:21,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95bf97a536a72d11a71086d876c972e2, server=24c191b0ea9d,43075,1732614319063 because future has completed 2024-11-26T09:45:21,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T09:45:21,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 95bf97a536a72d11a71086d876c972e2, server=24c191b0ea9d,43075,1732614319063 in 184 msec 2024-11-26T09:45:21,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T09:45:21,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, ASSIGN in 346 msec 2024-11-26T09:45:21,646 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732614321215 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T09:45:21,646 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732614321215","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732614321646"}]},"ts":"1732614321646"} 2024-11-26T09:45:21,649 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732614321215, state=ENABLED in hbase:meta 2024-11-26T09:45:21,650 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732614321215 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T09:45:21,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732614321215 in 409 msec 2024-11-26T09:45:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T09:45:21,891 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732614321215 completed 2024-11-26T09:45:21,891 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732614321215 get assigned. Timeout = 60000ms 2024-11-26T09:45:21,894 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T09:45:21,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732614321215 assigned to meta. Checking AM states. 2024-11-26T09:45:21,900 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T09:45:21,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732614321215 assigned. 2024-11-26T09:45:21,909 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732614321215', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2., hostname=24c191b0ea9d,43075,1732614319063, seqNum=2] 2024-11-26T09:45:21,921 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 95bf97a536a72d11a71086d876c972e2 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-26T09:45:21,947 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6357790e07774c98827e63aa64061ac0 2024-11-26T09:45:21,988 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/.tmp/family/6357790e07774c98827e63aa64061ac0 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-26T09:45:21,994 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/.tmp/family/6357790e07774c98827e63aa64061ac0 2024-11-26T09:45:22,033 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/.tmp/family/6357790e07774c98827e63aa64061ac0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0 2024-11-26T09:45:22,035 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0, delay=908 ms 2024-11-26T09:45:22,037 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0, entries=2, sequenceid=6, filesize=4.9 K 2024-11-26T09:45:22,043 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 95bf97a536a72d11a71086d876c972e2 in 121ms, sequenceid=6, compaction requested=false 2024-11-26T09:45:22,044 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732614321215' 2024-11-26T09:45:22,046 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 95bf97a536a72d11a71086d876c972e2: 2024-11-26T09:45:22,945 DEBUG [hfile-prefetch-1732614322035 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6357790e07774c98827e63aa64061ac0_0. 2024-11-26T09:45:22,949 DEBUG [hfile-prefetch-1732614322035 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6357790e07774c98827e63aa64061ac0_117. 2024-11-26T09:45:22,950 DEBUG [hfile-prefetch-1732614322035 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0, with total blocks 2, and data blocks 1 2024-11-26T09:45:22,950 DEBUG [hfile-prefetch-1732614322035 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T09:45:22,950 DEBUG [hfile-prefetch-1732614322035 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T09:45:22,950 DEBUG [hfile-prefetch-1732614322035 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0. Will sleep 100ms and try the verification again. 2024-11-26T09:45:22,957 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T09:45:22,957 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T09:45:22,957 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T09:45:22,957 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T09:45:22,977 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T09:45:22,977 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732614322957 2024-11-26T09:45:22,977 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T09:45:23,051 DEBUG [hfile-prefetch-1732614322035 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0, with total blocks 2, and data blocks 1 2024-11-26T09:45:23,051 DEBUG [hfile-prefetch-1732614322035 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-26T09:45:23,051 DEBUG [hfile-prefetch-1732614322035 {}] bucket.BucketCache(2293): found block for file 6357790e07774c98827e63aa64061ac0 in the backing map. Acquiring read lock for offset 0 2024-11-26T09:45:23,051 DEBUG [hfile-prefetch-1732614322035 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0 has now been fully cached. 2024-11-26T09:45:23,052 DEBUG [hfile-prefetch-1732614322035 {}] hfile.PrefetchExecutor(121): Prefetch completed for 6357790e07774c98827e63aa64061ac0 2024-11-26T09:45:23,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=95bf97a536a72d11a71086d876c972e2, source=24c191b0ea9d,43075,1732614319063, destination=24c191b0ea9d,41767,1732614319194, warming up region on 24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T09:45:23,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=95bf97a536a72d11a71086d876c972e2, source=24c191b0ea9d,43075,1732614319063, destination=24c191b0ea9d,41767,1732614319194, running balancer 2024-11-26T09:45:23,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42835, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T09:45:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, REOPEN/MOVE 2024-11-26T09:45:23,062 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, REOPEN/MOVE 2024-11-26T09:45:23,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41767 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41767 {}] regionserver.HRegion(7855): Warmup {ENCODED => 95bf97a536a72d11a71086d876c972e2, NAME => 'testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2.', STARTKEY => '', ENDKEY => ''} 2024-11-26T09:45:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41767 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:23,064 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=95bf97a536a72d11a71086d876c972e2, regionState=CLOSING, regionLocation=24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:23,065 INFO [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,066 INFO [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95bf97a536a72d11a71086d876c972e2 columnFamilyName family 2024-11-26T09:45:23,066 DEBUG [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:23,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, REOPEN/MOVE because future has completed 2024-11-26T09:45:23,067 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-26T09:45:23,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 95bf97a536a72d11a71086d876c972e2, server=24c191b0ea9d,43075,1732614319063}] 2024-11-26T09:45:23,071 DEBUG [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0 2024-11-26T09:45:23,071 INFO [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] regionserver.HStore(327): Store=95bf97a536a72d11a71086d876c972e2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41767 {}] regionserver.HRegion(1722): Closing 95bf97a536a72d11a71086d876c972e2, disabling compactions & flushes 2024-11-26T09:45:23,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41767 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41767 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41767 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. after waiting 0 ms 2024-11-26T09:45:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41767 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41767 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41767 {}] regionserver.HRegion(1676): Region close journal for 95bf97a536a72d11a71086d876c972e2: Waiting for close lock at 1732614323071Disabling compacts and flushes for region at 1732614323071Disabling writes for close at 1732614323071Writing region close event to WAL at 1732614323073 (+2 ms)Closed at 1732614323073 2024-11-26T09:45:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-26T09:45:23,226 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,226 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-26T09:45:23,227 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 95bf97a536a72d11a71086d876c972e2, disabling compactions & flushes 2024-11-26T09:45:23,227 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,227 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,227 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. after waiting 0 ms 2024-11-26T09:45:23,227 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,229 DEBUG [StoreFileCloser-95bf97a536a72d11a71086d876c972e2-family-1 {}] bucket.BucketCache(768): Removing file 6357790e07774c98827e63aa64061ac0 for region 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,229 DEBUG [StoreFileCloser-95bf97a536a72d11a71086d876c972e2-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 6357790e07774c98827e63aa64061ac0, starting offset: 0, end offset: 9223372036854775807 2024-11-26T09:45:23,230 DEBUG [StoreFileCloser-95bf97a536a72d11a71086d876c972e2-family-1 {}] bucket.BucketCache(856): removed key 6357790e07774c98827e63aa64061ac0_0 from back map with offset lock 2094080 in the evict process 2024-11-26T09:45:23,236 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-26T09:45:23,238 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,238 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 95bf97a536a72d11a71086d876c972e2: Waiting for close lock at 1732614323227Running coprocessor pre-close hooks at 1732614323227Disabling compacts and flushes for region at 1732614323227Disabling writes for close at 1732614323227Writing region close event to WAL at 1732614323231 (+4 ms)Running coprocessor post-close hooks at 1732614323236 (+5 ms)Closed at 1732614323238 (+2 ms) 2024-11-26T09:45:23,238 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 95bf97a536a72d11a71086d876c972e2 move to 24c191b0ea9d,41767,1732614319194 record at close sequenceid=6 2024-11-26T09:45:23,242 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,242 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=95bf97a536a72d11a71086d876c972e2, regionState=CLOSED 2024-11-26T09:45:23,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 95bf97a536a72d11a71086d876c972e2, server=24c191b0ea9d,43075,1732614319063 because future has completed 2024-11-26T09:45:23,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T09:45:23,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 95bf97a536a72d11a71086d876c972e2, server=24c191b0ea9d,43075,1732614319063 in 178 msec 2024-11-26T09:45:23,249 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, REOPEN/MOVE; state=CLOSED, location=24c191b0ea9d,41767,1732614319194; forceNewPlan=false, retain=false 2024-11-26T09:45:23,278 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T09:45:23,278 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T09:45:23,278 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T09:45:23,278 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T09:45:23,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T09:45:23,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732614323278 2024-11-26T09:45:23,282 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T09:45:23,399 INFO [24c191b0ea9d:36883 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T09:45:23,400 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=95bf97a536a72d11a71086d876c972e2, regionState=OPENING, regionLocation=24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:23,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, REOPEN/MOVE because future has completed 2024-11-26T09:45:23,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95bf97a536a72d11a71086d876c972e2, server=24c191b0ea9d,41767,1732614319194}] 2024-11-26T09:45:23,559 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,560 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 95bf97a536a72d11a71086d876c972e2, NAME => 'testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2.', STARTKEY => '', ENDKEY => ''} 2024-11-26T09:45:23,560 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732614321215 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,560 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:23,561 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,561 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,562 INFO [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,563 INFO [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95bf97a536a72d11a71086d876c972e2 columnFamilyName family 2024-11-26T09:45:23,563 DEBUG [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:23,565 DEBUG [StoreFileOpener-95bf97a536a72d11a71086d876c972e2-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0, delay=915 ms 2024-11-26T09:45:23,567 DEBUG [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0 2024-11-26T09:45:23,567 INFO [StoreOpener-95bf97a536a72d11a71086d876c972e2-1 {}] regionserver.HStore(327): Store=95bf97a536a72d11a71086d876c972e2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:23,567 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,568 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,568 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,569 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,569 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,573 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,574 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 95bf97a536a72d11a71086d876c972e2; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60530850, jitterRate=-0.0980200469493866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T09:45:23,574 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,575 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 95bf97a536a72d11a71086d876c972e2: Running coprocessor pre-open hook at 1732614323561Writing region info on filesystem at 1732614323561Initializing all the Stores at 1732614323561Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614323561Cleaning up temporary data from old regions at 1732614323569 (+8 ms)Running coprocessor post-open hooks at 1732614323574 (+5 ms)Region opened successfully at 1732614323575 (+1 ms) 2024-11-26T09:45:23,576 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2., pid=9, masterSystemTime=1732614323557 2024-11-26T09:45:23,578 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,578 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,579 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=95bf97a536a72d11a71086d876c972e2, regionState=OPEN, openSeqNum=10, regionLocation=24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:23,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95bf97a536a72d11a71086d876c972e2, server=24c191b0ea9d,41767,1732614319194 because future has completed 2024-11-26T09:45:23,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-26T09:45:23,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 95bf97a536a72d11a71086d876c972e2, server=24c191b0ea9d,41767,1732614319194 in 179 msec 2024-11-26T09:45:23,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732614321215, region=95bf97a536a72d11a71086d876c972e2, REOPEN/MOVE in 525 msec 2024-11-26T09:45:23,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T09:45:23,615 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T09:45:23,615 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T09:45:23,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:23,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:23,620 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T09:45:23,620 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T09:45:23,620 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-26T09:45:23,621 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=24c191b0ea9d,36883,1732614318310 2024-11-26T09:45:23,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:23,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:23,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:23,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:23,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:23,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:23,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T09:45:23,623 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T09:45:23,623 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T09:45:23,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:23,624 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '24c191b0ea9d,43075,1732614319063' ***** 2024-11-26T09:45:23,624 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T09:45:23,624 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '24c191b0ea9d,41767,1732614319194' ***** 2024-11-26T09:45:23,624 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T09:45:23,624 INFO [RS:0;24c191b0ea9d:43075 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T09:45:23,624 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:23,624 INFO [RS:1;24c191b0ea9d:41767 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T09:45:23,625 INFO [RS:1;24c191b0ea9d:41767 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/bucket.persistence 2024-11-26T09:45:23,625 INFO [RS:0;24c191b0ea9d:43075 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/bucket.persistence 2024-11-26T09:45:23,625 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T09:45:23,625 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:23,626 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T09:45:23,626 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T09:45:23,626 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T09:45:23,626 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T09:45:23,626 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T09:45:23,626 INFO [RS:1;24c191b0ea9d:41767 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T09:45:23,626 INFO [RS:0;24c191b0ea9d:43075 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T09:45:23,626 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:23,626 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T09:45:23,627 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T09:45:23,636 DEBUG [RS:1;24c191b0ea9d:41767 {}] bucket.BucketCache(1487): Thread RS:1;24c191b0ea9d:41767 started persisting bucket cache to file 2024-11-26T09:45:23,636 DEBUG [RS:0;24c191b0ea9d:43075 {}] bucket.BucketCache(1487): Thread RS:0;24c191b0ea9d:43075 started persisting bucket cache to file 2024-11-26T09:45:23,636 DEBUG [RS:0;24c191b0ea9d:43075 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T09:45:23,636 DEBUG [RS:1;24c191b0ea9d:41767 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T09:45:23,636 DEBUG [RS:1;24c191b0ea9d:41767 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T09:45:23,636 DEBUG [RS:0;24c191b0ea9d:43075 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T09:45:23,641 DEBUG [RS:0;24c191b0ea9d:43075 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T09:45:23,641 DEBUG [RS:0;24c191b0ea9d:43075 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732614323636 2024-11-26T09:45:23,641 DEBUG [RS:0;24c191b0ea9d:43075 {}] bucket.BucketCache(1509): Thread RS:0;24c191b0ea9d:43075 finished persisting bucket cache to file, renaming 2024-11-26T09:45:23,641 DEBUG [RS:1;24c191b0ea9d:41767 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T09:45:23,641 DEBUG [RS:1;24c191b0ea9d:41767 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732614323636 2024-11-26T09:45:23,641 DEBUG [RS:1;24c191b0ea9d:41767 {}] bucket.BucketCache(1509): Thread RS:1;24c191b0ea9d:41767 finished persisting bucket cache to file, renaming 2024-11-26T09:45:23,641 WARN [RS:1;24c191b0ea9d:41767 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-26T09:45:23,641 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T09:45:23,641 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T09:45:23,642 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T09:45:23,642 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T09:45:23,642 INFO [RS:1;24c191b0ea9d:41767 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T09:45:23,642 INFO [RS:0;24c191b0ea9d:43075 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T09:45:23,642 INFO [RS:0;24c191b0ea9d:43075 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T09:45:23,642 INFO [RS:1;24c191b0ea9d:41767 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T09:45:23,642 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(959): stopping server 24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:23,642 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(3091): Received CLOSE for 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,642 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T09:45:23,642 INFO [RS:0;24c191b0ea9d:43075 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;24c191b0ea9d:43075. 2024-11-26T09:45:23,642 DEBUG [RS:0;24c191b0ea9d:43075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T09:45:23,643 DEBUG [RS:0;24c191b0ea9d:43075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:23,643 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T09:45:23,643 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T09:45:23,643 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(959): stopping server 24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:23,643 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T09:45:23,643 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T09:45:23,643 INFO [RS:1;24c191b0ea9d:41767 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;24c191b0ea9d:41767. 2024-11-26T09:45:23,643 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T09:45:23,643 DEBUG [RS:1;24c191b0ea9d:41767 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T09:45:23,643 DEBUG [RS:1;24c191b0ea9d:41767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:23,643 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T09:45:23,643 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T09:45:23,643 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T09:45:23,644 DEBUG [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T09:45:23,644 DEBUG [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(1325): Online Regions={95bf97a536a72d11a71086d876c972e2=testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2.} 2024-11-26T09:45:23,644 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T09:45:23,644 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T09:45:23,644 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 95bf97a536a72d11a71086d876c972e2, disabling compactions & flushes 2024-11-26T09:45:23,644 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T09:45:23,644 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T09:45:23,644 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,644 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T09:45:23,644 DEBUG [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T09:45:23,644 DEBUG [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(1351): Waiting on 95bf97a536a72d11a71086d876c972e2 2024-11-26T09:45:23,644 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,644 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. after waiting 0 ms 2024-11-26T09:45:23,644 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,645 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T09:45:23,645 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-26T09:45:23,645 DEBUG [StoreFileCloser-95bf97a536a72d11a71086d876c972e2-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/family/6357790e07774c98827e63aa64061ac0 2024-11-26T09:45:23,651 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnRegionMove1732614321215/95bf97a536a72d11a71086d876c972e2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-26T09:45:23,651 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,651 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 95bf97a536a72d11a71086d876c972e2: Waiting for close lock at 1732614323643Running coprocessor pre-close hooks at 1732614323644 (+1 ms)Disabling compacts and flushes for region at 1732614323644Disabling writes for close at 1732614323644Writing region close event to WAL at 1732614323646 (+2 ms)Running coprocessor post-close hooks at 1732614323651 (+5 ms)Closed at 1732614323651 2024-11-26T09:45:23,651 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2. 2024-11-26T09:45:23,656 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 23d2a20aaa6944fa983779e71fea42e0 2024-11-26T09:45:23,665 INFO [regionserver/24c191b0ea9d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T09:45:23,666 INFO [regionserver/24c191b0ea9d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T09:45:23,674 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/info/23d2a20aaa6944fa983779e71fea42e0 is 205, key is testBlockEvictionOnRegionMove1732614321215,,1732614321232.95bf97a536a72d11a71086d876c972e2./info:regioninfo/1732614323579/Put/seqid=0 2024-11-26T09:45:23,675 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/info/23d2a20aaa6944fa983779e71fea42e0 2024-11-26T09:45:23,684 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 32d3c34470854e5c9444f6c46d78ca0f 2024-11-26T09:45:23,698 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/ns/32d3c34470854e5c9444f6c46d78ca0f is 43, key is default/ns:d/1732614321029/Put/seqid=0 2024-11-26T09:45:23,699 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/ns/32d3c34470854e5c9444f6c46d78ca0f 2024-11-26T09:45:23,709 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 311c647151c2486caea75d6548856374 2024-11-26T09:45:23,723 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/table/311c647151c2486caea75d6548856374 is 78, key is testBlockEvictionOnRegionMove1732614321215/table:state/1732614321646/Put/seqid=0 2024-11-26T09:45:23,723 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/table/311c647151c2486caea75d6548856374 2024-11-26T09:45:23,726 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/info/23d2a20aaa6944fa983779e71fea42e0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/23d2a20aaa6944fa983779e71fea42e0 2024-11-26T09:45:23,728 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/23d2a20aaa6944fa983779e71fea42e0, delay=929 ms 2024-11-26T09:45:23,729 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/23d2a20aaa6944fa983779e71fea42e0, entries=15, sequenceid=15, filesize=7.6 K 2024-11-26T09:45:23,730 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/ns/32d3c34470854e5c9444f6c46d78ca0f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/32d3c34470854e5c9444f6c46d78ca0f 2024-11-26T09:45:23,731 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/32d3c34470854e5c9444f6c46d78ca0f, delay=971 ms 2024-11-26T09:45:23,733 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/32d3c34470854e5c9444f6c46d78ca0f, entries=2, sequenceid=15, filesize=5.0 K 2024-11-26T09:45:23,733 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/table/311c647151c2486caea75d6548856374 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/311c647151c2486caea75d6548856374 2024-11-26T09:45:23,735 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/311c647151c2486caea75d6548856374, delay=976 ms 2024-11-26T09:45:23,736 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/311c647151c2486caea75d6548856374, entries=2, sequenceid=15, filesize=5.3 K 2024-11-26T09:45:23,737 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false 2024-11-26T09:45:23,737 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T09:45:23,738 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/23d2a20aaa6944fa983779e71fea42e0 2024-11-26T09:45:23,739 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/32d3c34470854e5c9444f6c46d78ca0f 2024-11-26T09:45:23,740 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/311c647151c2486caea75d6548856374 2024-11-26T09:45:23,745 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-26T09:45:23,745 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T09:45:23,745 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T09:45:23,746 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732614323644Running coprocessor pre-close hooks at 1732614323644Disabling compacts and flushes for region at 1732614323644Disabling writes for close at 1732614323644Obtaining lock to block concurrent updates at 1732614323645 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732614323645Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732614323645Flushing stores of hbase:meta,,1.1588230740 at 1732614323646 (+1 ms)Flushing 1588230740/info: creating writer at 1732614323646Flushing 1588230740/info: appending metadata at 1732614323673 (+27 ms)Flushing 1588230740/info: closing flushed file at 1732614323673Flushing 1588230740/ns: creating writer at 1732614323678 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732614323697 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732614323697Flushing 1588230740/table: creating writer at 1732614323702 (+5 ms)Flushing 1588230740/table: appending metadata at 1732614323722 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732614323722Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46a56101: reopening flushed file at 1732614323726 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f0e206a: reopening flushed file at 1732614323730 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c06cd5a: reopening flushed file at 1732614323733 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false at 1732614323737 (+4 ms)Writing region close event to WAL at 1732614323740 (+3 ms)Running coprocessor post-close hooks at 1732614323745 (+5 ms)Closed at 1732614323745 2024-11-26T09:45:23,746 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T09:45:23,845 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(976): stopping server 24c191b0ea9d,43075,1732614319063; all regions closed. 2024-11-26T09:45:23,845 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(976): stopping server 24c191b0ea9d,41767,1732614319194; all regions closed. 2024-11-26T09:45:23,850 DEBUG [RS:0;24c191b0ea9d:43075 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs 2024-11-26T09:45:23,850 DEBUG [RS:1;24c191b0ea9d:41767 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs 2024-11-26T09:45:23,850 INFO [RS:0;24c191b0ea9d:43075 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 24c191b0ea9d%2C43075%2C1732614319063.meta:.meta(num 1732614320900) 2024-11-26T09:45:23,850 INFO [RS:1;24c191b0ea9d:41767 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 24c191b0ea9d%2C41767%2C1732614319194:(num 1732614320523) 2024-11-26T09:45:23,850 DEBUG [RS:1;24c191b0ea9d:41767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:23,850 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T09:45:23,851 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T09:45:23,851 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.ChoreService(370): Chore service for: regionserver/24c191b0ea9d:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T09:45:23,851 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T09:45:23,851 INFO [regionserver/24c191b0ea9d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T09:45:23,851 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T09:45:23,851 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T09:45:23,852 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T09:45:23,852 INFO [RS:1;24c191b0ea9d:41767 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41767 2024-11-26T09:45:23,852 DEBUG [RS:0;24c191b0ea9d:43075 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs 2024-11-26T09:45:23,853 INFO [RS:0;24c191b0ea9d:43075 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 24c191b0ea9d%2C43075%2C1732614319063:(num 1732614320523) 2024-11-26T09:45:23,853 DEBUG [RS:0;24c191b0ea9d:43075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:23,853 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T09:45:23,853 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T09:45:23,853 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.ChoreService(370): Chore service for: regionserver/24c191b0ea9d:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T09:45:23,853 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T09:45:23,853 INFO [regionserver/24c191b0ea9d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T09:45:23,853 INFO [RS:0;24c191b0ea9d:43075 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43075 2024-11-26T09:45:23,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/24c191b0ea9d,41767,1732614319194 2024-11-26T09:45:23,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T09:45:23,856 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T09:45:23,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/24c191b0ea9d,43075,1732614319063 2024-11-26T09:45:23,857 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T09:45:23,859 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [24c191b0ea9d,43075,1732614319063] 2024-11-26T09:45:23,860 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/24c191b0ea9d,43075,1732614319063 already deleted, retry=false 2024-11-26T09:45:23,861 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 24c191b0ea9d,43075,1732614319063 expired; onlineServers=1 2024-11-26T09:45:23,861 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [24c191b0ea9d,41767,1732614319194] 2024-11-26T09:45:23,862 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/24c191b0ea9d,41767,1732614319194 already deleted, retry=false 2024-11-26T09:45:23,862 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 24c191b0ea9d,41767,1732614319194 expired; onlineServers=0 2024-11-26T09:45:23,862 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '24c191b0ea9d,36883,1732614318310' ***** 2024-11-26T09:45:23,862 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T09:45:23,862 INFO [M:0;24c191b0ea9d:36883 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T09:45:23,862 INFO [M:0;24c191b0ea9d:36883 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T09:45:23,862 DEBUG [M:0;24c191b0ea9d:36883 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T09:45:23,862 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T09:45:23,863 DEBUG [M:0;24c191b0ea9d:36883 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T09:45:23,863 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.small.0-1732614320169 {}] cleaner.HFileCleaner(306): Exit Thread[master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.small.0-1732614320169,5,FailOnTimeoutGroup] 2024-11-26T09:45:23,863 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.large.0-1732614320169 {}] cleaner.HFileCleaner(306): Exit Thread[master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.large.0-1732614320169,5,FailOnTimeoutGroup] 2024-11-26T09:45:23,863 INFO [M:0;24c191b0ea9d:36883 {}] hbase.ChoreService(370): Chore service for: master/24c191b0ea9d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T09:45:23,863 INFO [M:0;24c191b0ea9d:36883 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T09:45:23,863 DEBUG [M:0;24c191b0ea9d:36883 {}] master.HMaster(1795): Stopping service threads 2024-11-26T09:45:23,863 INFO [M:0;24c191b0ea9d:36883 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T09:45:23,863 INFO [M:0;24c191b0ea9d:36883 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T09:45:23,864 INFO [M:0;24c191b0ea9d:36883 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T09:45:23,864 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T09:45:23,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T09:45:23,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:23,865 DEBUG [M:0;24c191b0ea9d:36883 {}] zookeeper.ZKUtil(347): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T09:45:23,865 WARN [M:0;24c191b0ea9d:36883 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T09:45:23,865 INFO [M:0;24c191b0ea9d:36883 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/.lastflushedseqids 2024-11-26T09:45:23,874 INFO [M:0;24c191b0ea9d:36883 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T09:45:23,874 INFO [M:0;24c191b0ea9d:36883 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T09:45:23,875 DEBUG [M:0;24c191b0ea9d:36883 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T09:45:23,875 INFO [M:0;24c191b0ea9d:36883 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:23,875 DEBUG [M:0;24c191b0ea9d:36883 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:23,875 DEBUG [M:0;24c191b0ea9d:36883 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T09:45:23,875 DEBUG [M:0;24c191b0ea9d:36883 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:23,875 INFO [M:0;24c191b0ea9d:36883 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-26T09:45:23,886 DEBUG [M:0;24c191b0ea9d:36883 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4d62b2ab99544309a3fccaa73dc1b165 2024-11-26T09:45:23,899 DEBUG [M:0;24c191b0ea9d:36883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4d62b2ab99544309a3fccaa73dc1b165 is 82, key is hbase:meta,,1/info:regioninfo/1732614320964/Put/seqid=0 2024-11-26T09:45:23,900 INFO [M:0;24c191b0ea9d:36883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4d62b2ab99544309a3fccaa73dc1b165 2024-11-26T09:45:23,909 DEBUG [M:0;24c191b0ea9d:36883 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c4de94efe8704b6bacd1bdaeaeb8b566 2024-11-26T09:45:23,922 DEBUG [M:0;24c191b0ea9d:36883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c4de94efe8704b6bacd1bdaeaeb8b566 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732614321651/Put/seqid=0 2024-11-26T09:45:23,923 INFO [M:0;24c191b0ea9d:36883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c4de94efe8704b6bacd1bdaeaeb8b566 2024-11-26T09:45:23,933 DEBUG [M:0;24c191b0ea9d:36883 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6f7fd10a6cf94cae9e1c415768f24ffe 2024-11-26T09:45:23,946 DEBUG [M:0;24c191b0ea9d:36883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6f7fd10a6cf94cae9e1c415768f24ffe is 69, key is 24c191b0ea9d,41767,1732614319194/rs:state/1732614320275/Put/seqid=0 2024-11-26T09:45:23,947 INFO [M:0;24c191b0ea9d:36883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6f7fd10a6cf94cae9e1c415768f24ffe 2024-11-26T09:45:23,949 DEBUG [M:0;24c191b0ea9d:36883 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4d62b2ab99544309a3fccaa73dc1b165 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4d62b2ab99544309a3fccaa73dc1b165 2024-11-26T09:45:23,951 INFO [M:0;24c191b0ea9d:36883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4d62b2ab99544309a3fccaa73dc1b165, entries=8, sequenceid=84, filesize=5.5 K 2024-11-26T09:45:23,951 DEBUG [M:0;24c191b0ea9d:36883 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c4de94efe8704b6bacd1bdaeaeb8b566 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c4de94efe8704b6bacd1bdaeaeb8b566 2024-11-26T09:45:23,953 INFO [M:0;24c191b0ea9d:36883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c4de94efe8704b6bacd1bdaeaeb8b566, entries=9, sequenceid=84, filesize=6.5 K 2024-11-26T09:45:23,954 DEBUG [M:0;24c191b0ea9d:36883 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6f7fd10a6cf94cae9e1c415768f24ffe as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6f7fd10a6cf94cae9e1c415768f24ffe 2024-11-26T09:45:23,955 INFO [M:0;24c191b0ea9d:36883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6f7fd10a6cf94cae9e1c415768f24ffe, entries=2, sequenceid=84, filesize=5.1 K 2024-11-26T09:45:23,956 INFO [M:0;24c191b0ea9d:36883 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=84, compaction requested=false 2024-11-26T09:45:23,958 INFO [M:0;24c191b0ea9d:36883 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:23,959 DEBUG [M:0;24c191b0ea9d:36883 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732614323875Disabling compacts and flushes for region at 1732614323875Disabling writes for close at 1732614323875Obtaining lock to block concurrent updates at 1732614323875Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732614323875Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732614323876 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732614323876Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732614323876Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732614323899 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732614323899Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732614323902 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732614323922 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732614323922Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732614323925 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732614323946 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732614323946Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35f5df7b: reopening flushed file at 1732614323949 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@655047b1: reopening flushed file at 1732614323951 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ffc83c9: reopening flushed file at 1732614323953 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=84, compaction requested=false at 1732614323956 (+3 ms)Writing region close event to WAL at 1732614323958 (+2 ms)Closed at 1732614323958 2024-11-26T09:45:23,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:23,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41767-0x1003338209d0002, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:23,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:23,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1003338209d0001, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:23,960 INFO [RS:0;24c191b0ea9d:43075 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T09:45:23,960 INFO [RS:1;24c191b0ea9d:41767 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T09:45:23,960 INFO [RS:0;24c191b0ea9d:43075 {}] regionserver.HRegionServer(1031): Exiting; stopping=24c191b0ea9d,43075,1732614319063; zookeeper connection closed. 2024-11-26T09:45:23,960 INFO [RS:1;24c191b0ea9d:41767 {}] regionserver.HRegionServer(1031): Exiting; stopping=24c191b0ea9d,41767,1732614319194; zookeeper connection closed. 2024-11-26T09:45:23,960 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@42bf42d0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@42bf42d0 2024-11-26T09:45:23,961 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@abe2676 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@abe2676 2024-11-26T09:45:23,961 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T09:45:23,961 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-26T09:45:23,961 INFO [M:0;24c191b0ea9d:36883 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T09:45:23,961 INFO [M:0;24c191b0ea9d:36883 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36883 2024-11-26T09:45:23,962 INFO [M:0;24c191b0ea9d:36883 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T09:45:24,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:24,064 INFO [M:0;24c191b0ea9d:36883 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T09:45:24,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1003338209d0000, quorum=127.0.0.1:61297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:24,069 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T09:45:24,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T09:45:24,090 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:24,098 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/24c191b0ea9d:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/24c191b0ea9d:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732614323731 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732614323566 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/24c191b0ea9d:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732614323728 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732614322035 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/24c191b0ea9d:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1203 (was 1396), ProcessCount=11 (was 11), AvailableMemoryMB=4212 (was 4579) 2024-11-26T09:45:24,105 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=1203, ProcessCount=11, AvailableMemoryMB=4211 2024-11-26T09:45:24,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/hadoop-log-dir so I do NOT create it in target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81 2024-11-26T09:45:24,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/hadoop-tmp-dir so I do NOT create it in target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81 2024-11-26T09:45:24,105 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:24,109 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/cluster_716fe358-c711-74f8-a985-6a5a129cc340, deleteOnExit=true 2024-11-26T09:45:24,111 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/cluster_716fe358-c711-74f8-a985-6a5a129cc340/zookeeper_0, clientPort=63014, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/cluster_716fe358-c711-74f8-a985-6a5a129cc340/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/cluster_716fe358-c711-74f8-a985-6a5a129cc340/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T09:45:24,113 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63014 2024-11-26T09:45:24,125 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 with version=8 2024-11-26T09:45:24,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/hbase-staging 2024-11-26T09:45:24,128 INFO [Time-limited test {}] client.ConnectionUtils(128): master/24c191b0ea9d:0 server-side Connection retries=45 2024-11-26T09:45:24,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:24,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:24,128 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T09:45:24,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:24,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T09:45:24,128 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T09:45:24,129 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:24,129 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39635 2024-11-26T09:45:24,131 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39635 connecting to ZooKeeper ensemble=127.0.0.1:63014 2024-11-26T09:45:24,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:396350x0, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T09:45:24,137 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39635-0x100333838750000 connected 2024-11-26T09:45:24,152 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:24,152 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:24,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:24,153 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73, hbase.cluster.distributed=false 2024-11-26T09:45:24,155 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T09:45:24,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39635 2024-11-26T09:45:24,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39635 2024-11-26T09:45:24,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39635 2024-11-26T09:45:24,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39635 2024-11-26T09:45:24,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39635 2024-11-26T09:45:24,175 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/24c191b0ea9d:0 server-side Connection retries=45 2024-11-26T09:45:24,175 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:24,175 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:24,175 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T09:45:24,175 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:24,175 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T09:45:24,175 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T09:45:24,175 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:24,176 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34067 2024-11-26T09:45:24,178 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34067 connecting to ZooKeeper ensemble=127.0.0.1:63014 2024-11-26T09:45:24,178 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:24,178 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:24,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340670x0, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T09:45:24,182 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34067-0x100333838750001 connected 2024-11-26T09:45:24,182 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:24,182 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T09:45:24,183 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.cache 2024-11-26T09:45:24,183 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T09:45:24,183 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T09:45:24,183 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T09:45:24,184 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T09:45:24,184 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T09:45:24,184 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T09:45:24,184 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T09:45:24,184 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T09:45:24,184 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.persistence is complete. 2024-11-26T09:45:24,185 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T09:45:24,186 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T09:45:24,187 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34067 2024-11-26T09:45:24,187 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34067 2024-11-26T09:45:24,187 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34067 2024-11-26T09:45:24,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34067 2024-11-26T09:45:24,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34067 2024-11-26T09:45:24,205 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/24c191b0ea9d:0 server-side Connection retries=45 2024-11-26T09:45:24,205 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:24,205 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:24,205 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T09:45:24,205 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:24,205 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T09:45:24,205 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T09:45:24,205 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:24,206 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35317 2024-11-26T09:45:24,208 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35317 connecting to ZooKeeper ensemble=127.0.0.1:63014 2024-11-26T09:45:24,208 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:24,208 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:24,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:353170x0, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T09:45:24,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35317-0x100333838750002 connected 2024-11-26T09:45:24,212 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:24,212 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T09:45:24,213 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.cache 2024-11-26T09:45:24,213 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T09:45:24,213 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T09:45:24,213 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T09:45:24,214 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T09:45:24,214 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T09:45:24,214 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T09:45:24,214 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.persistence is complete. 2024-11-26T09:45:24,214 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T09:45:24,215 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T09:45:24,215 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T09:45:24,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T09:45:24,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35317 2024-11-26T09:45:24,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35317 2024-11-26T09:45:24,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35317 2024-11-26T09:45:24,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35317 2024-11-26T09:45:24,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35317 2024-11-26T09:45:24,239 DEBUG [M:0;24c191b0ea9d:39635 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;24c191b0ea9d:39635 2024-11-26T09:45:24,240 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:24,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:24,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:24,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:24,243 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:24,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T09:45:24,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T09:45:24,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,246 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T09:45:24,246 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/24c191b0ea9d,39635,1732614324127 from backup master directory 2024-11-26T09:45:24,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:24,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:24,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:24,248 WARN [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T09:45:24,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T09:45:24,248 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:24,249 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/hbase.id] with ID: ae8d1af5-c383-4390-aed7-a5519b55115d 2024-11-26T09:45:24,249 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/.tmp/hbase.id 2024-11-26T09:45:24,254 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/hbase.id] 2024-11-26T09:45:24,288 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:24,289 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T09:45:24,290 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-26T09:45:24,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,297 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T09:45:24,298 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T09:45:24,299 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T09:45:24,319 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store 2024-11-26T09:45:24,326 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:24,326 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T09:45:24,326 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:24,326 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:24,326 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T09:45:24,326 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:24,326 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:24,326 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732614324326Disabling compacts and flushes for region at 1732614324326Disabling writes for close at 1732614324326Writing region close event to WAL at 1732614324326Closed at 1732614324326 2024-11-26T09:45:24,327 WARN [master/24c191b0ea9d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/.initializing 2024-11-26T09:45:24,327 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/WALs/24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:24,332 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T09:45:24,336 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=24c191b0ea9d%2C39635%2C1732614324127, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/WALs/24c191b0ea9d,39635,1732614324127, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/oldWALs, maxLogs=10 2024-11-26T09:45:24,354 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/WALs/24c191b0ea9d,39635,1732614324127/24c191b0ea9d%2C39635%2C1732614324127.1732614324336 2024-11-26T09:45:24,355 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T09:45:24,355 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T09:45:24,355 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:24,355 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,355 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,356 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,360 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T09:45:24,361 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:24,361 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:24,361 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,364 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T09:45:24,364 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:24,364 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:24,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T09:45:24,368 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:24,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:24,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T09:45:24,372 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:24,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:24,372 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,373 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,373 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,373 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,373 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,373 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T09:45:24,374 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T09:45:24,380 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T09:45:24,381 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68424484, jitterRate=0.019604265689849854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T09:45:24,382 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732614324355Initializing all the Stores at 1732614324356 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614324356Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614324356Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614324356Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614324356Cleaning up temporary data from old regions at 1732614324373 (+17 ms)Region opened successfully at 1732614324382 (+9 ms) 2024-11-26T09:45:24,383 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T09:45:24,387 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a3edd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=24c191b0ea9d/172.17.0.2:0 2024-11-26T09:45:24,388 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T09:45:24,388 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T09:45:24,389 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T09:45:24,389 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T09:45:24,389 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T09:45:24,390 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-26T09:45:24,390 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T09:45:24,393 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T09:45:24,394 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T09:45:24,395 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T09:45:24,396 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T09:45:24,397 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T09:45:24,399 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T09:45:24,399 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T09:45:24,401 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T09:45:24,402 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T09:45:24,403 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T09:45:24,405 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T09:45:24,408 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T09:45:24,409 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T09:45:24,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:24,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:24,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:24,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,412 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=24c191b0ea9d,39635,1732614324127, sessionid=0x100333838750000, setting cluster-up flag (Was=false) 2024-11-26T09:45:24,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,420 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T09:45:24,422 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:24,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,429 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T09:45:24,431 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:24,431 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T09:45:24,433 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T09:45:24,433 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T09:45:24,433 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T09:45:24,434 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 24c191b0ea9d,39635,1732614324127 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T09:45:24,435 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/24c191b0ea9d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T09:45:24,435 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/24c191b0ea9d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T09:45:24,435 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/24c191b0ea9d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T09:45:24,435 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/24c191b0ea9d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T09:45:24,435 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/24c191b0ea9d:0, corePoolSize=10, maxPoolSize=10 2024-11-26T09:45:24,435 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,435 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/24c191b0ea9d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T09:45:24,435 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,439 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T09:45:24,440 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T09:45:24,440 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732614354440 2024-11-26T09:45:24,440 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T09:45:24,440 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T09:45:24,440 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T09:45:24,440 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T09:45:24,440 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:24,440 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T09:45:24,440 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T09:45:24,440 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T09:45:24,444 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,444 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T09:45:24,444 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T09:45:24,444 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T09:45:24,445 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T09:45:24,445 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T09:45:24,445 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.large.0-1732614324445,5,FailOnTimeoutGroup] 2024-11-26T09:45:24,445 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.small.0-1732614324445,5,FailOnTimeoutGroup] 2024-11-26T09:45:24,445 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,447 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T09:45:24,448 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,448 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,458 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T09:45:24,458 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 2024-11-26T09:45:24,467 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:24,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T09:45:24,472 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T09:45:24,472 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:24,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:24,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T09:45:24,477 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T09:45:24,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:24,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:24,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T09:45:24,481 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T09:45:24,481 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:24,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:24,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T09:45:24,485 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T09:45:24,486 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:24,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:24,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T09:45:24,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740 2024-11-26T09:45:24,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740 2024-11-26T09:45:24,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T09:45:24,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T09:45:24,488 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T09:45:24,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T09:45:24,496 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T09:45:24,497 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65694367, jitterRate=-0.021077647805213928}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T09:45:24,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732614324467Initializing all the Stores at 1732614324467Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614324467Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614324468 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614324468Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614324468Cleaning up temporary data from old regions at 1732614324488 (+20 ms)Region opened successfully at 1732614324498 (+10 ms) 2024-11-26T09:45:24,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T09:45:24,498 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T09:45:24,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T09:45:24,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T09:45:24,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T09:45:24,498 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T09:45:24,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732614324498Disabling compacts and flushes for region at 1732614324498Disabling writes for close at 1732614324498Writing region close event to WAL at 1732614324498Closed at 1732614324498 2024-11-26T09:45:24,499 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T09:45:24,500 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T09:45:24,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T09:45:24,501 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T09:45:24,502 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T09:45:24,523 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(746): ClusterId : ae8d1af5-c383-4390-aed7-a5519b55115d 2024-11-26T09:45:24,523 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(746): ClusterId : ae8d1af5-c383-4390-aed7-a5519b55115d 2024-11-26T09:45:24,523 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T09:45:24,523 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T09:45:24,525 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T09:45:24,525 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T09:45:24,525 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T09:45:24,526 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T09:45:24,528 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T09:45:24,529 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T09:45:24,529 DEBUG [RS:1;24c191b0ea9d:35317 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c9a8c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=24c191b0ea9d/172.17.0.2:0 2024-11-26T09:45:24,529 DEBUG [RS:0;24c191b0ea9d:34067 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a3ce03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=24c191b0ea9d/172.17.0.2:0 2024-11-26T09:45:24,544 DEBUG [RS:1;24c191b0ea9d:35317 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;24c191b0ea9d:35317 2024-11-26T09:45:24,544 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T09:45:24,544 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T09:45:24,545 DEBUG [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T09:45:24,545 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(2659): reportForDuty to master=24c191b0ea9d,39635,1732614324127 with port=35317, startcode=1732614324204 2024-11-26T09:45:24,546 DEBUG [RS:1;24c191b0ea9d:35317 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T09:45:24,548 DEBUG [RS:0;24c191b0ea9d:34067 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;24c191b0ea9d:34067 2024-11-26T09:45:24,548 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T09:45:24,548 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T09:45:24,548 DEBUG [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T09:45:24,548 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55153, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T09:45:24,549 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39635 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:24,549 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39635 {}] master.ServerManager(517): Registering regionserver=24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:24,549 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(2659): reportForDuty to master=24c191b0ea9d,39635,1732614324127 with port=34067, startcode=1732614324174 2024-11-26T09:45:24,549 DEBUG [RS:0;24c191b0ea9d:34067 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T09:45:24,551 DEBUG [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 2024-11-26T09:45:24,551 DEBUG [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T09:45:24,551 DEBUG [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T09:45:24,552 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35349, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T09:45:24,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T09:45:24,553 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39635 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:24,553 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39635 {}] master.ServerManager(517): Registering regionserver=24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:24,553 DEBUG [RS:1;24c191b0ea9d:35317 {}] zookeeper.ZKUtil(111): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:24,553 WARN [RS:1;24c191b0ea9d:35317 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T09:45:24,553 INFO [RS:1;24c191b0ea9d:35317 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T09:45:24,554 DEBUG [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:24,554 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [24c191b0ea9d,35317,1732614324204] 2024-11-26T09:45:24,554 DEBUG [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 2024-11-26T09:45:24,554 DEBUG [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T09:45:24,554 DEBUG [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T09:45:24,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T09:45:24,560 DEBUG [RS:0;24c191b0ea9d:34067 {}] zookeeper.ZKUtil(111): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:24,560 WARN [RS:0;24c191b0ea9d:34067 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T09:45:24,560 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [24c191b0ea9d,34067,1732614324174] 2024-11-26T09:45:24,560 INFO [RS:0;24c191b0ea9d:34067 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T09:45:24,560 DEBUG [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:24,560 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T09:45:24,564 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T09:45:24,564 INFO [RS:1;24c191b0ea9d:35317 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T09:45:24,565 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,565 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T09:45:24,565 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T09:45:24,566 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,566 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T09:45:24,566 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,566 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,566 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,566 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,566 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,566 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/24c191b0ea9d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T09:45:24,567 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,567 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,567 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,567 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,567 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,567 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,567 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T09:45:24,567 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/24c191b0ea9d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T09:45:24,567 DEBUG [RS:1;24c191b0ea9d:35317 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/24c191b0ea9d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T09:45:24,568 INFO [RS:0;24c191b0ea9d:34067 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T09:45:24,568 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,570 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T09:45:24,571 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,571 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,571 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,571 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,571 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,571 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,35317,1732614324204-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T09:45:24,571 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T09:45:24,571 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,571 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,571 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,571 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,571 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,572 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,572 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/24c191b0ea9d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T09:45:24,572 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,572 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,572 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,572 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,572 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,572 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:24,572 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/24c191b0ea9d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T09:45:24,572 DEBUG [RS:0;24c191b0ea9d:34067 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/24c191b0ea9d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T09:45:24,575 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,576 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,576 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,576 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,576 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,576 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,34067,1732614324174-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T09:45:24,588 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T09:45:24,588 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,35317,1732614324204-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,589 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,589 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.Replication(171): 24c191b0ea9d,35317,1732614324204 started 2024-11-26T09:45:24,592 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T09:45:24,592 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,34067,1732614324174-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,592 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,592 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.Replication(171): 24c191b0ea9d,34067,1732614324174 started 2024-11-26T09:45:24,603 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,603 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(1482): Serving as 24c191b0ea9d,35317,1732614324204, RpcServer on 24c191b0ea9d/172.17.0.2:35317, sessionid=0x100333838750002 2024-11-26T09:45:24,603 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T09:45:24,603 DEBUG [RS:1;24c191b0ea9d:35317 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:24,603 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '24c191b0ea9d,35317,1732614324204' 2024-11-26T09:45:24,604 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T09:45:24,604 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T09:45:24,605 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T09:45:24,605 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T09:45:24,605 DEBUG [RS:1;24c191b0ea9d:35317 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:24,605 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '24c191b0ea9d,35317,1732614324204' 2024-11-26T09:45:24,605 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T09:45:24,605 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T09:45:24,606 DEBUG [RS:1;24c191b0ea9d:35317 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T09:45:24,606 INFO [RS:1;24c191b0ea9d:35317 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T09:45:24,606 INFO [RS:1;24c191b0ea9d:35317 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T09:45:24,607 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:24,607 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(1482): Serving as 24c191b0ea9d,34067,1732614324174, RpcServer on 24c191b0ea9d/172.17.0.2:34067, sessionid=0x100333838750001 2024-11-26T09:45:24,607 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T09:45:24,607 DEBUG [RS:0;24c191b0ea9d:34067 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:24,607 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '24c191b0ea9d,34067,1732614324174' 2024-11-26T09:45:24,607 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T09:45:24,608 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T09:45:24,608 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T09:45:24,608 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T09:45:24,608 DEBUG [RS:0;24c191b0ea9d:34067 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:24,608 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '24c191b0ea9d,34067,1732614324174' 2024-11-26T09:45:24,608 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T09:45:24,609 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T09:45:24,609 DEBUG [RS:0;24c191b0ea9d:34067 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T09:45:24,609 INFO [RS:0;24c191b0ea9d:34067 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T09:45:24,609 INFO [RS:0;24c191b0ea9d:34067 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T09:45:24,652 WARN [24c191b0ea9d:39635 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T09:45:24,707 INFO [RS:1;24c191b0ea9d:35317 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T09:45:24,708 INFO [RS:1;24c191b0ea9d:35317 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=24c191b0ea9d%2C35317%2C1732614324204, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,35317,1732614324204, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs, maxLogs=55 2024-11-26T09:45:24,710 INFO [RS:0;24c191b0ea9d:34067 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T09:45:24,710 INFO [RS:0;24c191b0ea9d:34067 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=24c191b0ea9d%2C34067%2C1732614324174, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,34067,1732614324174, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs, maxLogs=55 2024-11-26T09:45:24,729 INFO [RS:1;24c191b0ea9d:35317 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,35317,1732614324204/24c191b0ea9d%2C35317%2C1732614324204.1732614324709 2024-11-26T09:45:24,729 INFO [RS:0;24c191b0ea9d:34067 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,34067,1732614324174/24c191b0ea9d%2C34067%2C1732614324174.1732614324712 2024-11-26T09:45:24,729 DEBUG [RS:1;24c191b0ea9d:35317 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T09:45:24,729 DEBUG [RS:0;24c191b0ea9d:34067 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T09:45:24,902 DEBUG [24c191b0ea9d:39635 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T09:45:24,903 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(204): Hosts are {24c191b0ea9d=0} racks are {/default-rack=0} 2024-11-26T09:45:24,904 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T09:45:24,904 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T09:45:24,904 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T09:45:24,904 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T09:45:24,904 INFO [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T09:45:24,904 INFO [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T09:45:24,904 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T09:45:24,904 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:24,906 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 24c191b0ea9d,35317,1732614324204, state=OPENING 2024-11-26T09:45:24,907 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T09:45:24,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:24,909 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T09:45:24,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=24c191b0ea9d,35317,1732614324204}] 2024-11-26T09:45:24,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:24,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:24,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:25,063 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T09:45:25,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53575, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T09:45:25,067 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T09:45:25,067 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T09:45:25,068 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T09:45:25,069 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=24c191b0ea9d%2C35317%2C1732614324204.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,35317,1732614324204, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs, maxLogs=55 2024-11-26T09:45:25,087 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,35317,1732614324204/24c191b0ea9d%2C35317%2C1732614324204.meta.1732614325069.meta 2024-11-26T09:45:25,087 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T09:45:25,088 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T09:45:25,088 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T09:45:25,088 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T09:45:25,089 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T09:45:25,089 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T09:45:25,089 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:25,089 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T09:45:25,089 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T09:45:25,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T09:45:25,092 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T09:45:25,092 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:25,093 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:25,093 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T09:45:25,094 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T09:45:25,094 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:25,094 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:25,094 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T09:45:25,095 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T09:45:25,095 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:25,095 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:25,095 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T09:45:25,096 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T09:45:25,096 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:25,096 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:25,096 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T09:45:25,097 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740 2024-11-26T09:45:25,097 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740 2024-11-26T09:45:25,098 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T09:45:25,098 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T09:45:25,098 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T09:45:25,099 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T09:45:25,099 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75056533, jitterRate=0.11842949688434601}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T09:45:25,099 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T09:45:25,100 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732614325089Writing region info on filesystem at 1732614325089Initializing all the Stores at 1732614325089Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614325089Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614325091 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614325091Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614325091Cleaning up temporary data from old regions at 1732614325098 (+7 ms)Running coprocessor post-open hooks at 1732614325099 (+1 ms)Region opened successfully at 1732614325100 (+1 ms) 2024-11-26T09:45:25,100 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732614325062 2024-11-26T09:45:25,103 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T09:45:25,103 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T09:45:25,104 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:25,105 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 24c191b0ea9d,35317,1732614324204, state=OPEN 2024-11-26T09:45:25,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:25,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:25,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:25,106 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:25,107 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:25,107 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:25,107 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:25,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T09:45:25,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=24c191b0ea9d,35317,1732614324204 in 197 msec 2024-11-26T09:45:25,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T09:45:25,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 609 msec 2024-11-26T09:45:25,112 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T09:45:25,112 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T09:45:25,116 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T09:45:25,116 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=24c191b0ea9d,35317,1732614324204, seqNum=-1] 2024-11-26T09:45:25,116 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T09:45:25,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50941, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T09:45:25,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 689 msec 2024-11-26T09:45:25,123 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732614325123, completionTime=-1 2024-11-26T09:45:25,123 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-26T09:45:25,123 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T09:45:25,125 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T09:45:25,125 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732614385125 2024-11-26T09:45:25,125 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732614445125 2024-11-26T09:45:25,126 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-26T09:45:25,126 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,39635,1732614324127-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:25,126 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,39635,1732614324127-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:25,126 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,39635,1732614324127-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:25,126 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-24c191b0ea9d:39635, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:25,126 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:25,126 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:25,129 DEBUG [master/24c191b0ea9d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T09:45:25,131 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.883sec 2024-11-26T09:45:25,131 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T09:45:25,131 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T09:45:25,132 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T09:45:25,132 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T09:45:25,132 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T09:45:25,132 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,39635,1732614324127-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T09:45:25,132 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,39635,1732614324127-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T09:45:25,133 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T09:45:25,133 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T09:45:25,133 INFO [master/24c191b0ea9d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,39635,1732614324127-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:25,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:25,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 24c191b0ea9d,39635,-1 for getting cluster id 2024-11-26T09:45:25,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T09:45:25,225 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ae8d1af5-c383-4390-aed7-a5519b55115d' 2024-11-26T09:45:25,226 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T09:45:25,226 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ae8d1af5-c383-4390-aed7-a5519b55115d" 2024-11-26T09:45:25,227 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b66929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:25,227 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [24c191b0ea9d,39635,-1] 2024-11-26T09:45:25,227 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T09:45:25,228 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:25,229 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T09:45:25,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:25,231 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T09:45:25,232 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=24c191b0ea9d,35317,1732614324204, seqNum=-1] 2024-11-26T09:45:25,233 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T09:45:25,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T09:45:25,237 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:25,237 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T09:45:25,238 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:25,238 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21b02fb2 2024-11-26T09:45:25,239 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T09:45:25,240 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T09:45:25,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39635 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732614325237', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T09:45:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39635 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732614325237 2024-11-26T09:45:25,244 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732614325237 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T09:45:25,244 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:25,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39635 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732614325237" procId is: 4 2024-11-26T09:45:25,245 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732614325237 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T09:45:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39635 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T09:45:25,257 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732614325237-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 80021d8852b8434751f6c9d34d6fa886, NAME => 'testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732614325237', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 2024-11-26T09:45:25,265 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732614325237-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:25,266 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732614325237-pool-0 {}] regionserver.HRegion(1722): Closing 80021d8852b8434751f6c9d34d6fa886, disabling compactions & flushes 2024-11-26T09:45:25,266 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732614325237-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:25,266 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732614325237-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:25,266 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732614325237-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. after waiting 0 ms 2024-11-26T09:45:25,266 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732614325237-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:25,266 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732614325237-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:25,266 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732614325237-pool-0 {}] regionserver.HRegion(1676): Region close journal for 80021d8852b8434751f6c9d34d6fa886: Waiting for close lock at 1732614325265Disabling compacts and flushes for region at 1732614325265Disabling writes for close at 1732614325266 (+1 ms)Writing region close event to WAL at 1732614325266Closed at 1732614325266 2024-11-26T09:45:25,267 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732614325237 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T09:45:25,267 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732614325267"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732614325267"}]},"ts":"1732614325267"} 2024-11-26T09:45:25,270 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T09:45:25,270 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732614325237 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T09:45:25,271 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732614325237","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732614325270"}]},"ts":"1732614325270"} 2024-11-26T09:45:25,273 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732614325237, state=ENABLING in hbase:meta 2024-11-26T09:45:25,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {24c191b0ea9d=0} racks are {/default-rack=0} 2024-11-26T09:45:25,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T09:45:25,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T09:45:25,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T09:45:25,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T09:45:25,273 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T09:45:25,273 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T09:45:25,273 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T09:45:25,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732614325237, region=80021d8852b8434751f6c9d34d6fa886, ASSIGN}] 2024-11-26T09:45:25,275 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732614325237, region=80021d8852b8434751f6c9d34d6fa886, ASSIGN 2024-11-26T09:45:25,275 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732614325237, region=80021d8852b8434751f6c9d34d6fa886, ASSIGN; state=OFFLINE, location=24c191b0ea9d,35317,1732614324204; forceNewPlan=false, retain=false 2024-11-26T09:45:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39635 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T09:45:25,426 INFO [24c191b0ea9d:39635 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T09:45:25,426 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=80021d8852b8434751f6c9d34d6fa886, regionState=OPENING, regionLocation=24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:25,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732614325237, region=80021d8852b8434751f6c9d34d6fa886, ASSIGN because future has completed 2024-11-26T09:45:25,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80021d8852b8434751f6c9d34d6fa886, server=24c191b0ea9d,35317,1732614324204}] 2024-11-26T09:45:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39635 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T09:45:25,585 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:25,585 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 80021d8852b8434751f6c9d34d6fa886, NAME => 'testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886.', STARTKEY => '', ENDKEY => ''} 2024-11-26T09:45:25,586 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732614325237 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,586 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:25,586 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,586 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,587 INFO [StoreOpener-80021d8852b8434751f6c9d34d6fa886-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,591 INFO [StoreOpener-80021d8852b8434751f6c9d34d6fa886-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80021d8852b8434751f6c9d34d6fa886 columnFamilyName family 2024-11-26T09:45:25,591 DEBUG [StoreOpener-80021d8852b8434751f6c9d34d6fa886-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:25,591 INFO [StoreOpener-80021d8852b8434751f6c9d34d6fa886-1 {}] regionserver.HStore(327): Store=80021d8852b8434751f6c9d34d6fa886/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:25,591 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,592 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,592 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,592 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,592 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,593 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,600 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T09:45:25,600 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 80021d8852b8434751f6c9d34d6fa886; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68823502, jitterRate=0.025550097227096558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T09:45:25,601 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:25,602 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 80021d8852b8434751f6c9d34d6fa886: Running coprocessor pre-open hook at 1732614325586Writing region info on filesystem at 1732614325586Initializing all the Stores at 1732614325586Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614325586Cleaning up temporary data from old regions at 1732614325593 (+7 ms)Running coprocessor post-open hooks at 1732614325601 (+8 ms)Region opened successfully at 1732614325601 2024-11-26T09:45:25,602 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886., pid=6, masterSystemTime=1732614325582 2024-11-26T09:45:25,605 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:25,605 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:25,605 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=80021d8852b8434751f6c9d34d6fa886, regionState=OPEN, openSeqNum=2, regionLocation=24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:25,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80021d8852b8434751f6c9d34d6fa886, server=24c191b0ea9d,35317,1732614324204 because future has completed 2024-11-26T09:45:25,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T09:45:25,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 80021d8852b8434751f6c9d34d6fa886, server=24c191b0ea9d,35317,1732614324204 in 179 msec 2024-11-26T09:45:25,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T09:45:25,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732614325237, region=80021d8852b8434751f6c9d34d6fa886, ASSIGN in 337 msec 2024-11-26T09:45:25,614 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732614325237 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T09:45:25,614 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732614325237","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732614325614"}]},"ts":"1732614325614"} 2024-11-26T09:45:25,616 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732614325237, state=ENABLED in hbase:meta 2024-11-26T09:45:25,617 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732614325237 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T09:45:25,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732614325237 in 375 msec 2024-11-26T09:45:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39635 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T09:45:25,870 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732614325237 completed 2024-11-26T09:45:25,870 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732614325237 get assigned. Timeout = 60000ms 2024-11-26T09:45:25,870 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T09:45:25,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732614325237 assigned to meta. Checking AM states. 2024-11-26T09:45:25,874 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T09:45:25,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732614325237 assigned. 2024-11-26T09:45:25,877 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732614325237', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886., hostname=24c191b0ea9d,35317,1732614324204, seqNum=2] 2024-11-26T09:45:25,882 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 80021d8852b8434751f6c9d34d6fa886 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-26T09:45:25,893 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 81c41ece5b194dfab68e17291b11cb0f 2024-11-26T09:45:25,906 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/.tmp/family/81c41ece5b194dfab68e17291b11cb0f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-26T09:45:25,907 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/.tmp/family/81c41ece5b194dfab68e17291b11cb0f 2024-11-26T09:45:25,910 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/.tmp/family/81c41ece5b194dfab68e17291b11cb0f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f 2024-11-26T09:45:25,911 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f, delay=950 ms 2024-11-26T09:45:25,912 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-26T09:45:25,913 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 80021d8852b8434751f6c9d34d6fa886 in 31ms, sequenceid=6, compaction requested=false 2024-11-26T09:45:25,913 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732614325237' 2024-11-26T09:45:25,913 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 80021d8852b8434751f6c9d34d6fa886: 2024-11-26T09:45:26,436 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T09:45:26,862 DEBUG [hfile-prefetch-1732614323566 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 81c41ece5b194dfab68e17291b11cb0f_0. 2024-11-26T09:45:26,863 DEBUG [hfile-prefetch-1732614323566 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 81c41ece5b194dfab68e17291b11cb0f_117. 2024-11-26T09:45:26,863 DEBUG [hfile-prefetch-1732614323566 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f, with total blocks 2, and data blocks 1 2024-11-26T09:45:26,863 DEBUG [hfile-prefetch-1732614323566 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T09:45:26,863 DEBUG [hfile-prefetch-1732614323566 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T09:45:26,863 DEBUG [hfile-prefetch-1732614323566 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f. Will sleep 100ms and try the verification again. 2024-11-26T09:45:26,885 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T09:45:26,885 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T09:45:26,886 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T09:45:26,886 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T09:45:26,889 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T09:45:26,889 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732614326886 2024-11-26T09:45:26,889 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T09:45:26,914 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;24c191b0ea9d:35317,5,FailOnTimeoutGroup] 2024-11-26T09:45:26,914 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '24c191b0ea9d,35317,1732614324204' ***** 2024-11-26T09:45:26,914 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-26T09:45:26,914 INFO [RS:1;24c191b0ea9d:35317 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T09:45:26,914 INFO [RS:1;24c191b0ea9d:35317 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.persistence 2024-11-26T09:45:26,914 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T09:45:26,914 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T09:45:26,914 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T09:45:26,915 INFO [RS:1;24c191b0ea9d:35317 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T09:45:26,915 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T09:45:26,925 DEBUG [RS:1;24c191b0ea9d:35317 {}] bucket.BucketCache(1487): Thread RS:1;24c191b0ea9d:35317 started persisting bucket cache to file 2024-11-26T09:45:26,925 DEBUG [RS:1;24c191b0ea9d:35317 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T09:45:26,925 DEBUG [RS:1;24c191b0ea9d:35317 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T09:45:26,928 DEBUG [RS:1;24c191b0ea9d:35317 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T09:45:26,928 DEBUG [RS:1;24c191b0ea9d:35317 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732614326925 2024-11-26T09:45:26,928 DEBUG [RS:1;24c191b0ea9d:35317 {}] bucket.BucketCache(1509): Thread RS:1;24c191b0ea9d:35317 finished persisting bucket cache to file, renaming 2024-11-26T09:45:26,929 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T09:45:26,929 INFO [RS:1;24c191b0ea9d:35317 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T09:45:26,929 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T09:45:26,929 INFO [RS:1;24c191b0ea9d:35317 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T09:45:26,929 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(3091): Received CLOSE for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:26,929 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(959): stopping server 24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:26,929 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T09:45:26,929 INFO [RS:1;24c191b0ea9d:35317 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;24c191b0ea9d:35317. 2024-11-26T09:45:26,929 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T09:45:26,929 DEBUG [RS:1;24c191b0ea9d:35317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T09:45:26,929 DEBUG [RS:1;24c191b0ea9d:35317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:26,930 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 80021d8852b8434751f6c9d34d6fa886, disabling compactions & flushes 2024-11-26T09:45:26,930 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:26,930 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T09:45:26,930 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:26,930 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T09:45:26,930 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T09:45:26,930 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. after waiting 0 ms 2024-11-26T09:45:26,930 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T09:45:26,930 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:26,930 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T09:45:26,930 DEBUG [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 80021d8852b8434751f6c9d34d6fa886=testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886.} 2024-11-26T09:45:26,930 DEBUG [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:26,930 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T09:45:26,930 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T09:45:26,930 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T09:45:26,930 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T09:45:26,931 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T09:45:26,931 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T09:45:26,931 DEBUG [StoreFileCloser-80021d8852b8434751f6c9d34d6fa886-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f 2024-11-26T09:45:26,931 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-26T09:45:26,933 WARN [hfile-prefetch-1732614323566 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-26T09:45:26,934 DEBUG [hfile-prefetch-1732614323566 {}] hfile.PrefetchExecutor(121): Prefetch completed for 81c41ece5b194dfab68e17291b11cb0f 2024-11-26T09:45:26,936 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-26T09:45:26,936 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:26,937 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 80021d8852b8434751f6c9d34d6fa886: Waiting for close lock at 1732614326929Running coprocessor pre-close hooks at 1732614326929Disabling compacts and flushes for region at 1732614326929Disabling writes for close at 1732614326930 (+1 ms)Writing region close event to WAL at 1732614326931 (+1 ms)Running coprocessor post-close hooks at 1732614326936 (+5 ms)Closed at 1732614326936 2024-11-26T09:45:26,937 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:26,942 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4f71f371822943db9edb0b11754f622d 2024-11-26T09:45:26,959 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/info/4f71f371822943db9edb0b11754f622d is 209, key is testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886./info:regioninfo/1732614325605/Put/seqid=0 2024-11-26T09:45:26,959 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/info/4f71f371822943db9edb0b11754f622d 2024-11-26T09:45:26,969 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 05a22d80375649009700558a89123c65 2024-11-26T09:45:26,973 INFO [regionserver/24c191b0ea9d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T09:45:26,983 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/ns/05a22d80375649009700558a89123c65 is 43, key is default/ns:d/1732614325118/Put/seqid=0 2024-11-26T09:45:26,984 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/ns/05a22d80375649009700558a89123c65 2024-11-26T09:45:26,993 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 89ffa314279d4ed3bc4914738eb1e1f2 2024-11-26T09:45:27,009 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/table/89ffa314279d4ed3bc4914738eb1e1f2 is 80, key is testBlockEvictionOnGracefulStop1732614325237/table:state/1732614325614/Put/seqid=0 2024-11-26T09:45:27,010 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/table/89ffa314279d4ed3bc4914738eb1e1f2 2024-11-26T09:45:27,013 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/info/4f71f371822943db9edb0b11754f622d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/4f71f371822943db9edb0b11754f622d 2024-11-26T09:45:27,015 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/4f71f371822943db9edb0b11754f622d, delay=933 ms 2024-11-26T09:45:27,016 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/4f71f371822943db9edb0b11754f622d, entries=10, sequenceid=11, filesize=7.0 K 2024-11-26T09:45:27,017 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/ns/05a22d80375649009700558a89123c65 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/05a22d80375649009700558a89123c65 2024-11-26T09:45:27,019 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/05a22d80375649009700558a89123c65, delay=905 ms 2024-11-26T09:45:27,020 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/05a22d80375649009700558a89123c65, entries=2, sequenceid=11, filesize=5.0 K 2024-11-26T09:45:27,021 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/table/89ffa314279d4ed3bc4914738eb1e1f2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/89ffa314279d4ed3bc4914738eb1e1f2 2024-11-26T09:45:27,022 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/89ffa314279d4ed3bc4914738eb1e1f2, delay=908 ms 2024-11-26T09:45:27,023 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/89ffa314279d4ed3bc4914738eb1e1f2, entries=2, sequenceid=11, filesize=5.3 K 2024-11-26T09:45:27,024 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false 2024-11-26T09:45:27,025 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/4f71f371822943db9edb0b11754f622d 2024-11-26T09:45:27,025 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/05a22d80375649009700558a89123c65 2024-11-26T09:45:27,026 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/89ffa314279d4ed3bc4914738eb1e1f2 2024-11-26T09:45:27,030 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-26T09:45:27,031 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T09:45:27,031 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T09:45:27,031 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732614326930Running coprocessor pre-close hooks at 1732614326930Disabling compacts and flushes for region at 1732614326930Disabling writes for close at 1732614326931 (+1 ms)Obtaining lock to block concurrent updates at 1732614326931Preparing flush snapshotting stores in 1588230740 at 1732614326931Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732614326931Flushing stores of hbase:meta,,1.1588230740 at 1732614326932 (+1 ms)Flushing 1588230740/info: creating writer at 1732614326932Flushing 1588230740/info: appending metadata at 1732614326958 (+26 ms)Flushing 1588230740/info: closing flushed file at 1732614326958Flushing 1588230740/ns: creating writer at 1732614326962 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732614326982 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732614326982Flushing 1588230740/table: creating writer at 1732614326987 (+5 ms)Flushing 1588230740/table: appending metadata at 1732614327008 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732614327008Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@152dfaaa: reopening flushed file at 1732614327013 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27849c10: reopening flushed file at 1732614327017 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15dfe617: reopening flushed file at 1732614327021 (+4 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false at 1732614327024 (+3 ms)Writing region close event to WAL at 1732614327026 (+2 ms)Running coprocessor post-close hooks at 1732614327031 (+5 ms)Closed at 1732614327031 2024-11-26T09:45:27,031 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T09:45:27,130 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(976): stopping server 24c191b0ea9d,35317,1732614324204; all regions closed. 2024-11-26T09:45:27,133 DEBUG [RS:1;24c191b0ea9d:35317 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs 2024-11-26T09:45:27,133 INFO [RS:1;24c191b0ea9d:35317 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 24c191b0ea9d%2C35317%2C1732614324204.meta:.meta(num 1732614325069) 2024-11-26T09:45:27,135 DEBUG [RS:1;24c191b0ea9d:35317 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs 2024-11-26T09:45:27,135 INFO [RS:1;24c191b0ea9d:35317 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 24c191b0ea9d%2C35317%2C1732614324204:(num 1732614324709) 2024-11-26T09:45:27,135 DEBUG [RS:1;24c191b0ea9d:35317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:27,135 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T09:45:27,135 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T09:45:27,135 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.ChoreService(370): Chore service for: regionserver/24c191b0ea9d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T09:45:27,135 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T09:45:27,135 INFO [regionserver/24c191b0ea9d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T09:45:27,135 INFO [RS:1;24c191b0ea9d:35317 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35317 2024-11-26T09:45:27,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/24c191b0ea9d,35317,1732614324204 2024-11-26T09:45:27,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T09:45:27,140 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T09:45:27,141 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [24c191b0ea9d,35317,1732614324204] 2024-11-26T09:45:27,143 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/24c191b0ea9d,35317,1732614324204 already deleted, retry=false 2024-11-26T09:45:27,143 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 24c191b0ea9d,35317,1732614324204 on 24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:27,148 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 24c191b0ea9d,35317,1732614324204, splitWal=true, meta=true 2024-11-26T09:45:27,150 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 24c191b0ea9d,35317,1732614324204 (carryingMeta=true) 24c191b0ea9d,35317,1732614324204/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@e4b20c7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T09:45:27,150 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 24c191b0ea9d,35317,1732614324204, splitWal=true, meta=true 2024-11-26T09:45:27,151 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 24c191b0ea9d,35317,1732614324204, splitWal=true, meta=true, isMeta: true 2024-11-26T09:45:27,151 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,35317,1732614324204-splitting 2024-11-26T09:45:27,152 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,35317,1732614324204-splitting dir is empty, no logs to split. 2024-11-26T09:45:27,152 INFO [PEWorker-2 {}] master.SplitWALManager(105): 24c191b0ea9d,35317,1732614324204 WAL count=0, meta=true 2024-11-26T09:45:27,153 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,35317,1732614324204-splitting dir is empty, no logs to split. 2024-11-26T09:45:27,153 INFO [PEWorker-2 {}] master.SplitWALManager(105): 24c191b0ea9d,35317,1732614324204 WAL count=0, meta=true 2024-11-26T09:45:27,153 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 24c191b0ea9d,35317,1732614324204 WAL splitting is done? wals=0, meta=true 2024-11-26T09:45:27,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T09:45:27,156 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T09:45:27,156 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T09:45:27,242 INFO [RS:1;24c191b0ea9d:35317 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T09:45:27,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:27,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35317-0x100333838750002, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:27,242 INFO [RS:1;24c191b0ea9d:35317 {}] regionserver.HRegionServer(1031): Exiting; stopping=24c191b0ea9d,35317,1732614324204; zookeeper connection closed. 2024-11-26T09:45:27,243 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2c954ed2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2c954ed2 2024-11-26T09:45:27,307 DEBUG [24c191b0ea9d:39635 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T09:45:27,307 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:27,308 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 24c191b0ea9d,34067,1732614324174, state=OPENING 2024-11-26T09:45:27,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:27,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:27,310 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T09:45:27,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:27,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:27,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=24c191b0ea9d,34067,1732614324174}] 2024-11-26T09:45:27,431 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/24c191b0ea9d:0 server-side Connection retries=45 2024-11-26T09:45:27,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:27,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:27,431 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T09:45:27,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T09:45:27,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T09:45:27,431 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T09:45:27,431 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:27,432 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43241 2024-11-26T09:45:27,433 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43241 connecting to ZooKeeper ensemble=127.0.0.1:63014 2024-11-26T09:45:27,434 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:27,434 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:27,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432410x0, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T09:45:27,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43241-0x100333838750003 connected 2024-11-26T09:45:27,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-26T09:45:27,438 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-26T09:45:27,438 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T09:45:27,439 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.cache 2024-11-26T09:45:27,439 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T09:45:27,439 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T09:45:27,439 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T09:45:27,440 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T09:45:27,440 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T09:45:27,440 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T09:45:27,440 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-26T09:45:27,441 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T09:45:27,442 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-26T09:45:27,442 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T09:45:27,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43241 2024-11-26T09:45:27,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43241 2024-11-26T09:45:27,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43241 2024-11-26T09:45:27,445 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43241 2024-11-26T09:45:27,445 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-26T09:45:27,445 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43241 2024-11-26T09:45:27,446 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T09:45:27,446 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-26T09:45:27,446 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.persistence is complete. 2024-11-26T09:45:27,446 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(746): ClusterId : ae8d1af5-c383-4390-aed7-a5519b55115d 2024-11-26T09:45:27,446 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T09:45:27,449 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T09:45:27,449 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T09:45:27,451 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T09:45:27,451 DEBUG [RS:2;24c191b0ea9d:43241 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c97e571, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=24c191b0ea9d/172.17.0.2:0 2024-11-26T09:45:27,463 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T09:45:27,465 DEBUG [RS:2;24c191b0ea9d:43241 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;24c191b0ea9d:43241 2024-11-26T09:45:27,465 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T09:45:27,465 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T09:45:27,465 DEBUG [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T09:45:27,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38685, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T09:45:27,466 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(2659): reportForDuty to master=24c191b0ea9d,39635,1732614324127 with port=43241, startcode=1732614327430 2024-11-26T09:45:27,466 DEBUG [RS:2;24c191b0ea9d:43241 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T09:45:27,467 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T09:45:27,468 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T09:45:27,468 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T09:45:27,468 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32813, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T09:45:27,469 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39635 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 24c191b0ea9d,43241,1732614327430 2024-11-26T09:45:27,469 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39635 {}] master.ServerManager(517): Registering regionserver=24c191b0ea9d,43241,1732614327430 2024-11-26T09:45:27,469 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=24c191b0ea9d%2C34067%2C1732614324174.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,34067,1732614324174, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs, maxLogs=55 2024-11-26T09:45:27,470 DEBUG [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73 2024-11-26T09:45:27,470 DEBUG [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T09:45:27,470 DEBUG [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T09:45:27,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T09:45:27,473 DEBUG [RS:2;24c191b0ea9d:43241 {}] zookeeper.ZKUtil(111): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/24c191b0ea9d,43241,1732614327430 2024-11-26T09:45:27,473 WARN [RS:2;24c191b0ea9d:43241 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T09:45:27,473 INFO [RS:2;24c191b0ea9d:43241 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T09:45:27,473 DEBUG [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,43241,1732614327430 2024-11-26T09:45:27,473 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [24c191b0ea9d,43241,1732614327430] 2024-11-26T09:45:27,479 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T09:45:27,482 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T09:45:27,483 INFO [RS:2;24c191b0ea9d:43241 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T09:45:27,483 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:27,483 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T09:45:27,484 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T09:45:27,484 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/24c191b0ea9d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,484 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/24c191b0ea9d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T09:45:27,485 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/24c191b0ea9d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T09:45:27,485 DEBUG [RS:2;24c191b0ea9d:43241 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/24c191b0ea9d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T09:45:27,486 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:27,486 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:27,486 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:27,486 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:27,486 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:27,486 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,43241,1732614327430-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T09:45:27,492 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,34067,1732614324174/24c191b0ea9d%2C34067%2C1732614324174.meta.1732614327470.meta 2024-11-26T09:45:27,492 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T09:45:27,492 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T09:45:27,493 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T09:45:27,493 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T09:45:27,493 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T09:45:27,493 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T09:45:27,494 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:27,494 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T09:45:27,494 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T09:45:27,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T09:45:27,497 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T09:45:27,497 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:27,501 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/4f71f371822943db9edb0b11754f622d, delay=997 ms 2024-11-26T09:45:27,502 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/4f71f371822943db9edb0b11754f622d 2024-11-26T09:45:27,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:27,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T09:45:27,503 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T09:45:27,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:27,505 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/05a22d80375649009700558a89123c65, delay=978 ms 2024-11-26T09:45:27,506 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/05a22d80375649009700558a89123c65 2024-11-26T09:45:27,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:27,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T09:45:27,507 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T09:45:27,507 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:27,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:27,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T09:45:27,508 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T09:45:27,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:27,509 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/89ffa314279d4ed3bc4914738eb1e1f2, delay=904 ms 2024-11-26T09:45:27,510 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T09:45:27,510 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=24c191b0ea9d,43241,1732614327430-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:27,510 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:27,510 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.Replication(171): 24c191b0ea9d,43241,1732614327430 started 2024-11-26T09:45:27,510 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/89ffa314279d4ed3bc4914738eb1e1f2 2024-11-26T09:45:27,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T09:45:27,511 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T09:45:27,511 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740 2024-11-26T09:45:27,512 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740 2024-11-26T09:45:27,512 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T09:45:27,512 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T09:45:27,513 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T09:45:27,514 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T09:45:27,515 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66142018, jitterRate=-0.014407128095626831}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T09:45:27,515 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T09:45:27,516 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732614327494Writing region info on filesystem at 1732614327494Initializing all the Stores at 1732614327494Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614327494Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614327496 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614327496Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732614327496Cleaning up temporary data from old regions at 1732614327512 (+16 ms)Running coprocessor post-open hooks at 1732614327515 (+3 ms)Region opened successfully at 1732614327516 (+1 ms) 2024-11-26T09:45:27,517 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732614327462 2024-11-26T09:45:27,520 DEBUG [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T09:45:27,520 INFO [RS_OPEN_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T09:45:27,520 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:27,521 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 24c191b0ea9d,34067,1732614324174, state=OPEN 2024-11-26T09:45:27,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:27,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:27,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T09:45:27,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:27,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:27,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T09:45:27,523 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:27,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-26T09:45:27,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=24c191b0ea9d,34067,1732614324174 in 213 msec 2024-11-26T09:45:27,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T09:45:27,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 370 msec 2024-11-26T09:45:27,527 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 24c191b0ea9d,35317,1732614324204 had 2 regions 2024-11-26T09:45:27,528 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 24c191b0ea9d,35317,1732614324204, splitWal=true, meta=true, isMeta: false 2024-11-26T09:45:27,528 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,35317,1732614324204-splitting dir is empty, no logs to split. 2024-11-26T09:45:27,528 INFO [PEWorker-3 {}] master.SplitWALManager(105): 24c191b0ea9d,35317,1732614324204 WAL count=0, meta=false 2024-11-26T09:45:27,529 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,35317,1732614324204-splitting dir is empty, no logs to split. 2024-11-26T09:45:27,530 INFO [PEWorker-3 {}] master.SplitWALManager(105): 24c191b0ea9d,35317,1732614324204 WAL count=0, meta=false 2024-11-26T09:45:27,530 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 24c191b0ea9d,35317,1732614324204 WAL splitting is done? wals=0, meta=false 2024-11-26T09:45:27,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732614325237, region=80021d8852b8434751f6c9d34d6fa886, ASSIGN}] 2024-11-26T09:45:27,532 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732614325237, region=80021d8852b8434751f6c9d34d6fa886, ASSIGN 2024-11-26T09:45:27,533 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732614325237, region=80021d8852b8434751f6c9d34d6fa886, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T09:45:27,533 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T09:45:27,533 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(1482): Serving as 24c191b0ea9d,43241,1732614327430, RpcServer on 24c191b0ea9d/172.17.0.2:43241, sessionid=0x100333838750003 2024-11-26T09:45:27,534 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T09:45:27,534 DEBUG [RS:2;24c191b0ea9d:43241 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 24c191b0ea9d,43241,1732614327430 2024-11-26T09:45:27,534 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '24c191b0ea9d,43241,1732614327430' 2024-11-26T09:45:27,534 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T09:45:27,535 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T09:45:27,535 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T09:45:27,535 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T09:45:27,535 DEBUG [RS:2;24c191b0ea9d:43241 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 24c191b0ea9d,43241,1732614327430 2024-11-26T09:45:27,535 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '24c191b0ea9d,43241,1732614327430' 2024-11-26T09:45:27,535 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T09:45:27,536 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T09:45:27,536 DEBUG [RS:2;24c191b0ea9d:43241 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T09:45:27,537 INFO [RS:2;24c191b0ea9d:43241 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T09:45:27,537 INFO [RS:2;24c191b0ea9d:43241 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T09:45:27,637 INFO [RS:2;24c191b0ea9d:43241 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T09:45:27,638 INFO [RS:2;24c191b0ea9d:43241 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=24c191b0ea9d%2C43241%2C1732614327430, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,43241,1732614327430, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs, maxLogs=55 2024-11-26T09:45:27,665 INFO [RS:2;24c191b0ea9d:43241 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/WALs/24c191b0ea9d,43241,1732614327430/24c191b0ea9d%2C43241%2C1732614327430.1732614327639 2024-11-26T09:45:27,665 DEBUG [RS:2;24c191b0ea9d:43241 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T09:45:27,684 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(204): Hosts are {24c191b0ea9d=0} racks are {/default-rack=0} 2024-11-26T09:45:27,684 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T09:45:27,684 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T09:45:27,684 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T09:45:27,684 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T09:45:27,684 INFO [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T09:45:27,684 INFO [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T09:45:27,684 DEBUG [24c191b0ea9d:39635 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T09:45:27,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=80021d8852b8434751f6c9d34d6fa886, regionState=OPENING, regionLocation=24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:27,686 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 24c191b0ea9d:35317 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 24c191b0ea9d/172.17.0.2:35317 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:45:27,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=24c191b0ea9d,35317,1732614324204, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=24c191b0ea9d,35317,1732614324204, seqNum=-1, error=java.net.ConnectException: Call to address=24c191b0ea9d:35317 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 24c191b0ea9d/172.17.0.2:35317 2024-11-26T09:45:27,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=24c191b0ea9d,35317,1732614324204, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-26T09:45:27,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=24c191b0ea9d,35317,1732614324204, seqNum=-1 from cache 2024-11-26T09:45:27,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 24c191b0ea9d:35317 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 24c191b0ea9d/172.17.0.2:35317 2024-11-26T09:45:27,799 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T09:45:27,800 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=24c191b0ea9d,34067,1732614324174, seqNum=-1] 2024-11-26T09:45:27,800 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T09:45:27,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46783, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T09:45:27,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732614325237, region=80021d8852b8434751f6c9d34d6fa886, ASSIGN because future has completed 2024-11-26T09:45:27,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80021d8852b8434751f6c9d34d6fa886, server=24c191b0ea9d,34067,1732614324174}] 2024-11-26T09:45:27,960 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:27,960 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 80021d8852b8434751f6c9d34d6fa886, NAME => 'testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886.', STARTKEY => '', ENDKEY => ''} 2024-11-26T09:45:27,961 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732614325237 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,961 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:45:27,961 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,961 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,962 INFO [StoreOpener-80021d8852b8434751f6c9d34d6fa886-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,963 INFO [StoreOpener-80021d8852b8434751f6c9d34d6fa886-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80021d8852b8434751f6c9d34d6fa886 columnFamilyName family 2024-11-26T09:45:27,963 DEBUG [StoreOpener-80021d8852b8434751f6c9d34d6fa886-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:45:27,964 DEBUG [StoreFileOpener-80021d8852b8434751f6c9d34d6fa886-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f, delay=946 ms 2024-11-26T09:45:27,965 DEBUG [StoreOpener-80021d8852b8434751f6c9d34d6fa886-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f 2024-11-26T09:45:27,966 INFO [StoreOpener-80021d8852b8434751f6c9d34d6fa886-1 {}] regionserver.HStore(327): Store=80021d8852b8434751f6c9d34d6fa886/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:45:27,966 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,966 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,967 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,967 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,967 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,968 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,968 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 80021d8852b8434751f6c9d34d6fa886; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63133702, jitterRate=-0.05923452973365784}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T09:45:27,969 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:27,970 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 80021d8852b8434751f6c9d34d6fa886: Running coprocessor pre-open hook at 1732614327961Writing region info on filesystem at 1732614327961Initializing all the Stores at 1732614327962 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732614327962Cleaning up temporary data from old regions at 1732614327967 (+5 ms)Running coprocessor post-open hooks at 1732614327969 (+2 ms)Region opened successfully at 1732614327970 (+1 ms) 2024-11-26T09:45:27,970 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886., pid=11, masterSystemTime=1732614327957 2024-11-26T09:45:27,973 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=80021d8852b8434751f6c9d34d6fa886, regionState=OPEN, openSeqNum=10, regionLocation=24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:27,974 DEBUG [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:27,974 INFO [RS_OPEN_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:27,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80021d8852b8434751f6c9d34d6fa886, server=24c191b0ea9d,34067,1732614324174 because future has completed 2024-11-26T09:45:27,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-26T09:45:27,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 80021d8852b8434751f6c9d34d6fa886, server=24c191b0ea9d,34067,1732614324174 in 172 msec 2024-11-26T09:45:27,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-26T09:45:27,980 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 24c191b0ea9d,35317,1732614324204 after splitting done 2024-11-26T09:45:27,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732614325237, region=80021d8852b8434751f6c9d34d6fa886, ASSIGN in 447 msec 2024-11-26T09:45:27,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 24c191b0ea9d,35317,1732614324204, splitWal=true, meta=true in 836 msec 2024-11-26T09:45:28,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T09:45:28,034 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T09:45:28,034 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T09:45:28,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:28,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:28,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T09:45:28,035 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T09:45:28,035 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=116358662, stopped=false 2024-11-26T09:45:28,035 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=24c191b0ea9d,39635,1732614324127 2024-11-26T09:45:28,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:28,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:28,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T09:45:28,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:28,037 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T09:45:28,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:28,037 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T09:45:28,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:28,038 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T09:45:28,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:28,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:28,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:28,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T09:45:28,038 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '24c191b0ea9d,34067,1732614324174' ***** 2024-11-26T09:45:28,038 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T09:45:28,039 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '24c191b0ea9d,43241,1732614327430' ***** 2024-11-26T09:45:28,039 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T09:45:28,039 INFO [RS:0;24c191b0ea9d:34067 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T09:45:28,039 INFO [RS:0;24c191b0ea9d:34067 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.persistence 2024-11-26T09:45:28,039 INFO [RS:2;24c191b0ea9d:43241 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T09:45:28,039 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T09:45:28,039 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T09:45:28,039 INFO [RS:2;24c191b0ea9d:43241 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6fc4e1b4-47b6-e409-98bb-819c63a58a81/bucket.persistence 2024-11-26T09:45:28,039 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T09:45:28,039 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T09:45:28,039 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T09:45:28,039 INFO [RS:0;24c191b0ea9d:34067 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T09:45:28,039 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T09:45:28,040 INFO [RS:2;24c191b0ea9d:43241 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T09:45:28,040 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T09:45:28,040 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T09:45:28,050 DEBUG [RS:0;24c191b0ea9d:34067 {}] bucket.BucketCache(1487): Thread RS:0;24c191b0ea9d:34067 started persisting bucket cache to file 2024-11-26T09:45:28,050 DEBUG [RS:0;24c191b0ea9d:34067 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T09:45:28,050 DEBUG [RS:0;24c191b0ea9d:34067 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T09:45:28,050 DEBUG [RS:2;24c191b0ea9d:43241 {}] bucket.BucketCache(1487): Thread RS:2;24c191b0ea9d:43241 started persisting bucket cache to file 2024-11-26T09:45:28,050 DEBUG [RS:2;24c191b0ea9d:43241 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T09:45:28,050 DEBUG [RS:2;24c191b0ea9d:43241 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T09:45:28,053 DEBUG [RS:0;24c191b0ea9d:34067 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T09:45:28,053 DEBUG [RS:0;24c191b0ea9d:34067 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732614328050 2024-11-26T09:45:28,053 DEBUG [RS:2;24c191b0ea9d:43241 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T09:45:28,053 DEBUG [RS:2;24c191b0ea9d:43241 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732614328050 2024-11-26T09:45:28,053 DEBUG [RS:0;24c191b0ea9d:34067 {}] bucket.BucketCache(1509): Thread RS:0;24c191b0ea9d:34067 finished persisting bucket cache to file, renaming 2024-11-26T09:45:28,054 DEBUG [RS:2;24c191b0ea9d:43241 {}] bucket.BucketCache(1509): Thread RS:2;24c191b0ea9d:43241 finished persisting bucket cache to file, renaming 2024-11-26T09:45:28,054 WARN [RS:2;24c191b0ea9d:43241 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-26T09:45:28,054 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T09:45:28,054 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T09:45:28,054 INFO [RS:2;24c191b0ea9d:43241 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T09:45:28,054 INFO [RS:0;24c191b0ea9d:34067 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T09:45:28,054 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T09:45:28,054 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T09:45:28,054 INFO [RS:2;24c191b0ea9d:43241 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T09:45:28,054 INFO [RS:0;24c191b0ea9d:34067 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T09:45:28,054 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(959): stopping server 24c191b0ea9d,43241,1732614327430 2024-11-26T09:45:28,054 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T09:45:28,054 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(3091): Received CLOSE for 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:28,054 INFO [RS:2;24c191b0ea9d:43241 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;24c191b0ea9d:43241. 2024-11-26T09:45:28,054 DEBUG [RS:2;24c191b0ea9d:43241 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T09:45:28,054 DEBUG [RS:2;24c191b0ea9d:43241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:28,054 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(976): stopping server 24c191b0ea9d,43241,1732614327430; all regions closed. 2024-11-26T09:45:28,054 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(959): stopping server 24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:28,054 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T09:45:28,055 INFO [RS:0;24c191b0ea9d:34067 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;24c191b0ea9d:34067. 2024-11-26T09:45:28,055 DEBUG [RS:0;24c191b0ea9d:34067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T09:45:28,055 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T09:45:28,055 DEBUG [RS:0;24c191b0ea9d:34067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:28,055 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 80021d8852b8434751f6c9d34d6fa886, disabling compactions & flushes 2024-11-26T09:45:28,055 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T09:45:28,055 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:28,055 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T09:45:28,055 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T09:45:28,055 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:28,055 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T09:45:28,055 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. after waiting 0 ms 2024-11-26T09:45:28,055 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:28,055 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T09:45:28,055 DEBUG [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 80021d8852b8434751f6c9d34d6fa886=testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886.} 2024-11-26T09:45:28,055 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T09:45:28,055 DEBUG [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 80021d8852b8434751f6c9d34d6fa886 2024-11-26T09:45:28,056 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T09:45:28,056 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T09:45:28,056 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T09:45:28,056 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T09:45:28,056 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T09:45:28,056 DEBUG [StoreFileCloser-80021d8852b8434751f6c9d34d6fa886-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/family/81c41ece5b194dfab68e17291b11cb0f 2024-11-26T09:45:28,056 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-26T09:45:28,056 DEBUG [RS:2;24c191b0ea9d:43241 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs 2024-11-26T09:45:28,056 INFO [RS:2;24c191b0ea9d:43241 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 24c191b0ea9d%2C43241%2C1732614327430:(num 1732614327639) 2024-11-26T09:45:28,056 DEBUG [RS:2;24c191b0ea9d:43241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:28,056 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T09:45:28,057 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T09:45:28,057 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.ChoreService(370): Chore service for: regionserver/24c191b0ea9d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T09:45:28,057 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T09:45:28,057 INFO [regionserver/24c191b0ea9d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T09:45:28,057 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T09:45:28,057 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T09:45:28,057 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T09:45:28,057 INFO [RS:2;24c191b0ea9d:43241 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43241 2024-11-26T09:45:28,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/24c191b0ea9d,43241,1732614327430 2024-11-26T09:45:28,059 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T09:45:28,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T09:45:28,061 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [24c191b0ea9d,43241,1732614327430] 2024-11-26T09:45:28,062 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/default/testBlockEvictionOnGracefulStop1732614325237/80021d8852b8434751f6c9d34d6fa886/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-26T09:45:28,062 INFO [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:28,062 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 80021d8852b8434751f6c9d34d6fa886: Waiting for close lock at 1732614328055Running coprocessor pre-close hooks at 1732614328055Disabling compacts and flushes for region at 1732614328055Disabling writes for close at 1732614328055Writing region close event to WAL at 1732614328056 (+1 ms)Running coprocessor post-close hooks at 1732614328062 (+6 ms)Closed at 1732614328062 2024-11-26T09:45:28,062 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/24c191b0ea9d,43241,1732614327430 already deleted, retry=false 2024-11-26T09:45:28,063 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 24c191b0ea9d,43241,1732614327430 expired; onlineServers=1 2024-11-26T09:45:28,063 DEBUG [RS_CLOSE_REGION-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886. 2024-11-26T09:45:28,067 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1b4fa8eb5c9e4b2da5f0e866aecce9f9 2024-11-26T09:45:28,080 INFO [regionserver/24c191b0ea9d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T09:45:28,081 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/info/1b4fa8eb5c9e4b2da5f0e866aecce9f9 is 209, key is testBlockEvictionOnGracefulStop1732614325237,,1732614325241.80021d8852b8434751f6c9d34d6fa886./info:regioninfo/1732614327973/Put/seqid=0 2024-11-26T09:45:28,082 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/info/1b4fa8eb5c9e4b2da5f0e866aecce9f9 2024-11-26T09:45:28,084 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/.tmp/info/1b4fa8eb5c9e4b2da5f0e866aecce9f9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/1b4fa8eb5c9e4b2da5f0e866aecce9f9 2024-11-26T09:45:28,085 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/1b4fa8eb5c9e4b2da5f0e866aecce9f9, delay=960 ms 2024-11-26T09:45:28,086 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/1b4fa8eb5c9e4b2da5f0e866aecce9f9, entries=8, sequenceid=19, filesize=6.6 K 2024-11-26T09:45:28,087 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false 2024-11-26T09:45:28,087 INFO [regionserver/24c191b0ea9d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T09:45:28,088 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/4f71f371822943db9edb0b11754f622d 2024-11-26T09:45:28,088 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/info/1b4fa8eb5c9e4b2da5f0e866aecce9f9 2024-11-26T09:45:28,089 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/ns/05a22d80375649009700558a89123c65 2024-11-26T09:45:28,089 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/table/89ffa314279d4ed3bc4914738eb1e1f2 2024-11-26T09:45:28,094 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-26T09:45:28,094 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T09:45:28,094 INFO [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T09:45:28,094 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732614328055Running coprocessor pre-close hooks at 1732614328055Disabling compacts and flushes for region at 1732614328056 (+1 ms)Disabling writes for close at 1732614328056Obtaining lock to block concurrent updates at 1732614328056Preparing flush snapshotting stores in 1588230740 at 1732614328056Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732614328056Flushing stores of hbase:meta,,1.1588230740 at 1732614328057 (+1 ms)Flushing 1588230740/info: creating writer at 1732614328057Flushing 1588230740/info: appending metadata at 1732614328081 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732614328081Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69a1b42b: reopening flushed file at 1732614328084 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false at 1732614328087 (+3 ms)Writing region close event to WAL at 1732614328090 (+3 ms)Running coprocessor post-close hooks at 1732614328094 (+4 ms)Closed at 1732614328094 2024-11-26T09:45:28,094 DEBUG [RS_CLOSE_META-regionserver/24c191b0ea9d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T09:45:28,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:28,161 INFO [RS:2;24c191b0ea9d:43241 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T09:45:28,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43241-0x100333838750003, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:28,161 INFO [RS:2;24c191b0ea9d:43241 {}] regionserver.HRegionServer(1031): Exiting; stopping=24c191b0ea9d,43241,1732614327430; zookeeper connection closed. 2024-11-26T09:45:28,162 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4bacf699 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4bacf699 2024-11-26T09:45:28,256 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(976): stopping server 24c191b0ea9d,34067,1732614324174; all regions closed. 2024-11-26T09:45:28,258 DEBUG [RS:0;24c191b0ea9d:34067 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs 2024-11-26T09:45:28,258 INFO [RS:0;24c191b0ea9d:34067 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 24c191b0ea9d%2C34067%2C1732614324174.meta:.meta(num 1732614327470) 2024-11-26T09:45:28,260 DEBUG [RS:0;24c191b0ea9d:34067 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/oldWALs 2024-11-26T09:45:28,260 INFO [RS:0;24c191b0ea9d:34067 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 24c191b0ea9d%2C34067%2C1732614324174:(num 1732614324712) 2024-11-26T09:45:28,260 DEBUG [RS:0;24c191b0ea9d:34067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:28,260 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T09:45:28,260 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T09:45:28,260 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.ChoreService(370): Chore service for: regionserver/24c191b0ea9d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T09:45:28,260 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T09:45:28,260 INFO [regionserver/24c191b0ea9d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T09:45:28,260 INFO [RS:0;24c191b0ea9d:34067 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34067 2024-11-26T09:45:28,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/24c191b0ea9d,34067,1732614324174 2024-11-26T09:45:28,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T09:45:28,263 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T09:45:28,264 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [24c191b0ea9d,34067,1732614324174] 2024-11-26T09:45:28,266 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/24c191b0ea9d,34067,1732614324174 already deleted, retry=false 2024-11-26T09:45:28,266 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 24c191b0ea9d,34067,1732614324174 expired; onlineServers=0 2024-11-26T09:45:28,266 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '24c191b0ea9d,39635,1732614324127' ***** 2024-11-26T09:45:28,266 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T09:45:28,266 INFO [M:0;24c191b0ea9d:39635 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T09:45:28,266 INFO [M:0;24c191b0ea9d:39635 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T09:45:28,266 DEBUG [M:0;24c191b0ea9d:39635 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T09:45:28,266 DEBUG [M:0;24c191b0ea9d:39635 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T09:45:28,266 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T09:45:28,266 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.large.0-1732614324445 {}] cleaner.HFileCleaner(306): Exit Thread[master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.large.0-1732614324445,5,FailOnTimeoutGroup] 2024-11-26T09:45:28,266 DEBUG [master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.small.0-1732614324445 {}] cleaner.HFileCleaner(306): Exit Thread[master/24c191b0ea9d:0:becomeActiveMaster-HFileCleaner.small.0-1732614324445,5,FailOnTimeoutGroup] 2024-11-26T09:45:28,266 INFO [M:0;24c191b0ea9d:39635 {}] hbase.ChoreService(370): Chore service for: master/24c191b0ea9d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T09:45:28,267 INFO [M:0;24c191b0ea9d:39635 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T09:45:28,267 DEBUG [M:0;24c191b0ea9d:39635 {}] master.HMaster(1795): Stopping service threads 2024-11-26T09:45:28,267 INFO [M:0;24c191b0ea9d:39635 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T09:45:28,267 INFO [M:0;24c191b0ea9d:39635 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T09:45:28,267 INFO [M:0;24c191b0ea9d:39635 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T09:45:28,267 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T09:45:28,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T09:45:28,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T09:45:28,268 DEBUG [M:0;24c191b0ea9d:39635 {}] zookeeper.ZKUtil(347): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T09:45:28,268 WARN [M:0;24c191b0ea9d:39635 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T09:45:28,269 INFO [M:0;24c191b0ea9d:39635 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/.lastflushedseqids 2024-11-26T09:45:28,276 INFO [M:0;24c191b0ea9d:39635 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T09:45:28,276 INFO [M:0;24c191b0ea9d:39635 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T09:45:28,276 DEBUG [M:0;24c191b0ea9d:39635 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T09:45:28,276 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:28,276 DEBUG [M:0;24c191b0ea9d:39635 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:28,276 DEBUG [M:0;24c191b0ea9d:39635 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T09:45:28,276 DEBUG [M:0;24c191b0ea9d:39635 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:28,276 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-26T09:45:28,286 DEBUG [M:0;24c191b0ea9d:39635 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f57df0a7742c41d39e83f2a41804d905 2024-11-26T09:45:28,300 DEBUG [M:0;24c191b0ea9d:39635 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f57df0a7742c41d39e83f2a41804d905 is 82, key is hbase:meta,,1/info:regioninfo/1732614327520/Put/seqid=0 2024-11-26T09:45:28,300 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f57df0a7742c41d39e83f2a41804d905 2024-11-26T09:45:28,309 DEBUG [M:0;24c191b0ea9d:39635 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0e286ecafb1840c084467d4a9ccb80a9 2024-11-26T09:45:28,322 DEBUG [M:0;24c191b0ea9d:39635 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0e286ecafb1840c084467d4a9ccb80a9 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732614325618/Put/seqid=0 2024-11-26T09:45:28,323 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0e286ecafb1840c084467d4a9ccb80a9 2024-11-26T09:45:28,331 DEBUG [M:0;24c191b0ea9d:39635 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8e2f3bfd162f44e79215e4a762ef1e5e 2024-11-26T09:45:28,344 DEBUG [M:0;24c191b0ea9d:39635 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8e2f3bfd162f44e79215e4a762ef1e5e is 69, key is 24c191b0ea9d,34067,1732614324174/rs:state/1732614324553/Put/seqid=0 2024-11-26T09:45:28,344 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8e2f3bfd162f44e79215e4a762ef1e5e 2024-11-26T09:45:28,346 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8e2f3bfd162f44e79215e4a762ef1e5e 2024-11-26T09:45:28,347 DEBUG [M:0;24c191b0ea9d:39635 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f57df0a7742c41d39e83f2a41804d905 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f57df0a7742c41d39e83f2a41804d905 2024-11-26T09:45:28,348 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f57df0a7742c41d39e83f2a41804d905, entries=14, sequenceid=102, filesize=5.9 K 2024-11-26T09:45:28,348 DEBUG [M:0;24c191b0ea9d:39635 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0e286ecafb1840c084467d4a9ccb80a9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0e286ecafb1840c084467d4a9ccb80a9 2024-11-26T09:45:28,350 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0e286ecafb1840c084467d4a9ccb80a9, entries=11, sequenceid=102, filesize=6.5 K 2024-11-26T09:45:28,350 DEBUG [M:0;24c191b0ea9d:39635 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8e2f3bfd162f44e79215e4a762ef1e5e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8e2f3bfd162f44e79215e4a762ef1e5e 2024-11-26T09:45:28,351 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8e2f3bfd162f44e79215e4a762ef1e5e 2024-11-26T09:45:28,351 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/03ef4c32-cd65-ffe4-27e5-0d04075d9786/aa0b855e-88c6-ff95-9e50-02a1457a6b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8e2f3bfd162f44e79215e4a762ef1e5e, entries=3, sequenceid=102, filesize=5.3 K 2024-11-26T09:45:28,352 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=102, compaction requested=false 2024-11-26T09:45:28,354 INFO [M:0;24c191b0ea9d:39635 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T09:45:28,354 DEBUG [M:0;24c191b0ea9d:39635 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732614328276Disabling compacts and flushes for region at 1732614328276Disabling writes for close at 1732614328276Obtaining lock to block concurrent updates at 1732614328276Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732614328276Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1732614328277 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732614328277Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732614328277Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732614328299 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732614328299Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732614328302 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732614328322 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732614328322Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732614328325 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732614328343 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732614328343Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c5a5ebe: reopening flushed file at 1732614328346 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13223276: reopening flushed file at 1732614328348 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2432d25f: reopening flushed file at 1732614328350 (+2 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=102, compaction requested=false at 1732614328352 (+2 ms)Writing region close event to WAL at 1732614328354 (+2 ms)Closed at 1732614328354 2024-11-26T09:45:28,355 INFO [M:0;24c191b0ea9d:39635 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T09:45:28,355 INFO [M:0;24c191b0ea9d:39635 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39635 2024-11-26T09:45:28,355 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T09:45:28,355 INFO [M:0;24c191b0ea9d:39635 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T09:45:28,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:28,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34067-0x100333838750001, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:28,364 INFO [RS:0;24c191b0ea9d:34067 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T09:45:28,365 INFO [RS:0;24c191b0ea9d:34067 {}] regionserver.HRegionServer(1031): Exiting; stopping=24c191b0ea9d,34067,1732614324174; zookeeper connection closed. 2024-11-26T09:45:28,365 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@272ae557 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@272ae557 2024-11-26T09:45:28,365 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T09:45:28,458 INFO [M:0;24c191b0ea9d:39635 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T09:45:28,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:28,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39635-0x100333838750000, quorum=127.0.0.1:63014, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T09:45:28,461 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T09:45:28,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T09:45:28,463 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T09:45:28,470 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1203 (was 1203), ProcessCount=11 (was 11), AvailableMemoryMB=4096 (was 4211)