2024-11-12 07:50:47,319 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-12 07:50:47,344 main DEBUG Took 0.021988 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 07:50:47,345 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 07:50:47,345 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 07:50:47,347 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 07:50:47,348 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,357 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 07:50:47,373 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,376 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,377 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,378 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,378 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,379 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,379 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,380 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,381 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,382 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,383 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,384 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,385 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,386 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,386 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,387 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,387 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,388 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,389 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:50:47,389 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,390 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 07:50:47,392 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:50:47,394 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 07:50:47,396 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 07:50:47,397 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 07:50:47,399 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 07:50:47,400 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 07:50:47,411 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 07:50:47,414 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 07:50:47,417 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 07:50:47,417 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 07:50:47,418 main DEBUG createAppenders(={Console}) 2024-11-12 07:50:47,419 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-12 07:50:47,420 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-12 07:50:47,420 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-12 07:50:47,421 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 07:50:47,421 main DEBUG OutputStream closed 2024-11-12 07:50:47,422 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 07:50:47,422 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 07:50:47,422 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-12 07:50:47,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 07:50:47,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 07:50:47,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 07:50:47,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 07:50:47,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 07:50:47,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 07:50:47,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 07:50:47,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 07:50:47,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 07:50:47,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 07:50:47,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 07:50:47,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 07:50:47,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 07:50:47,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 07:50:47,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 07:50:47,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 07:50:47,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 07:50:47,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 07:50:47,530 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 07:50:47,531 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-12 07:50:47,531 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 07:50:47,532 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-12T07:50:47,544 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-12 07:50:47,547 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 07:50:47,547 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T07:50:47,834 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828 2024-11-12T07:50:47,862 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=394, ProcessCount=11, AvailableMemoryMB=7969 2024-11-12T07:50:47,969 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T07:50:48,072 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:48,088 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/cluster_b2635f47-8edb-f280-3f17-4f9307eb8d8b, deleteOnExit=true 2024-11-12T07:50:48,227 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/cluster_b2635f47-8edb-f280-3f17-4f9307eb8d8b/zookeeper_0, clientPort=56168, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/cluster_b2635f47-8edb-f280-3f17-4f9307eb8d8b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/cluster_b2635f47-8edb-f280-3f17-4f9307eb8d8b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T07:50:48,238 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56168 2024-11-12T07:50:48,239 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:48,355 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 with version=8 2024-11-12T07:50:48,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/hbase-staging 2024-11-12T07:50:48,488 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T07:50:48,840 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f9fc96254ef8:0 server-side Connection retries=45 2024-11-12T07:50:48,850 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:48,852 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:48,863 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:50:48,864 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:48,864 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:50:49,026 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T07:50:49,121 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:50:49,182 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T07:50:49,197 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T07:50:49,202 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:50:49,249 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 36825 (auto-detected) 2024-11-12T07:50:49,251 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T07:50:49,289 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43613 2024-11-12T07:50:49,320 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43613 connecting to ZooKeeper ensemble=127.0.0.1:56168 2024-11-12T07:50:49,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:436130x0, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:50:49,435 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43613-0x1012f501f500000 connected 2024-11-12T07:50:49,527 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:49,529 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:49,538 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:49,542 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6, hbase.cluster.distributed=false 2024-11-12T07:50:49,591 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:50:49,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43613 2024-11-12T07:50:49,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43613 2024-11-12T07:50:49,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43613 2024-11-12T07:50:49,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43613 2024-11-12T07:50:49,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43613 2024-11-12T07:50:49,739 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f9fc96254ef8:0 server-side Connection retries=45 2024-11-12T07:50:49,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:49,742 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:49,742 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:50:49,742 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:49,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:50:49,747 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T07:50:49,751 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:50:49,752 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36017 2024-11-12T07:50:49,755 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36017 connecting to ZooKeeper ensemble=127.0.0.1:56168 2024-11-12T07:50:49,756 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:49,757 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:49,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360170x0, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:50:49,795 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360170x0, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:49,796 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36017-0x1012f501f500001 connected 2024-11-12T07:50:49,801 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T07:50:49,812 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/bucket.cache 2024-11-12T07:50:49,818 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T07:50:49,819 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T07:50:49,821 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T07:50:49,823 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T07:50:49,823 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T07:50:49,829 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T07:50:49,836 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T07:50:49,836 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/bucket.persistence is complete. 2024-11-12T07:50:49,838 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T07:50:49,842 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T07:50:49,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:50:49,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36017 2024-11-12T07:50:49,855 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36017 2024-11-12T07:50:49,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36017 2024-11-12T07:50:49,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36017 2024-11-12T07:50:49,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36017 2024-11-12T07:50:49,888 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f9fc96254ef8:0 server-side Connection retries=45 2024-11-12T07:50:49,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:49,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:49,889 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:50:49,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:49,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:50:49,890 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T07:50:49,891 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:50:49,892 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38735 2024-11-12T07:50:49,895 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38735 connecting to ZooKeeper ensemble=127.0.0.1:56168 2024-11-12T07:50:49,896 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:49,897 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:49,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387350x0, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:50:49,957 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387350x0, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:49,958 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T07:50:49,959 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38735-0x1012f501f500002 connected 2024-11-12T07:50:49,960 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/bucket.cache 2024-11-12T07:50:49,960 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T07:50:49,961 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T07:50:49,961 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T07:50:49,962 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T07:50:49,962 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T07:50:49,962 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T07:50:49,963 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T07:50:49,963 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T07:50:49,963 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/bucket.persistence is complete. 2024-11-12T07:50:49,964 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T07:50:49,966 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:50:49,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38735 2024-11-12T07:50:49,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38735 2024-11-12T07:50:49,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38735 2024-11-12T07:50:49,981 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38735 2024-11-12T07:50:49,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38735 2024-11-12T07:50:49,999 DEBUG [M:0;f9fc96254ef8:43613 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f9fc96254ef8:43613 2024-11-12T07:50:50,001 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f9fc96254ef8,43613,1731397848605 2024-11-12T07:50:50,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:50,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:50,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:50,032 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f9fc96254ef8,43613,1731397848605 2024-11-12T07:50:50,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T07:50:50,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T07:50:50,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:50,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:50,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:50,074 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T07:50:50,075 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f9fc96254ef8,43613,1731397848605 from backup master directory 2024-11-12T07:50:50,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f9fc96254ef8,43613,1731397848605 2024-11-12T07:50:50,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:50,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:50,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:50,091 WARN [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:50:50,091 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f9fc96254ef8,43613,1731397848605 2024-11-12T07:50:50,094 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T07:50:50,097 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T07:50:50,108 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/hbase.id] with ID: d6b65e10-50df-47c5-b58b-900b97c93d69 2024-11-12T07:50:50,109 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/.tmp/hbase.id 2024-11-12T07:50:50,114 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/hbase.id] 2024-11-12T07:50:50,158 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:50,163 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T07:50:50,182 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-12T07:50:50,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:50,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:50,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:50,257 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:50:50,261 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T07:50:50,306 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:50,313 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:50:50,375 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store 2024-11-12T07:50:50,401 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T07:50:50,405 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:50,407 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T07:50:50,407 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:50,408 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:50,410 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T07:50:50,410 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:50,411 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:50,411 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731397850407Disabling compacts and flushes for region at 1731397850407Disabling writes for close at 1731397850410 (+3 ms)Writing region close event to WAL at 1731397850410Closed at 1731397850410 2024-11-12T07:50:50,412 WARN [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/.initializing 2024-11-12T07:50:50,412 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/WALs/f9fc96254ef8,43613,1731397848605 2024-11-12T07:50:50,431 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:50:50,447 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f9fc96254ef8%2C43613%2C1731397848605, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/WALs/f9fc96254ef8,43613,1731397848605, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/oldWALs, maxLogs=10 2024-11-12T07:50:50,498 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/WALs/f9fc96254ef8,43613,1731397848605/f9fc96254ef8%2C43613%2C1731397848605.1731397850453 2024-11-12T07:50:50,500 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T07:50:50,500 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:50,502 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:50,506 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,508 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,558 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,590 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T07:50:50,595 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:50,596 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:50,597 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T07:50:50,603 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:50,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:50,604 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T07:50:50,610 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:50,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:50,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T07:50:50,619 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:50,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:50,620 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,623 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,624 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,627 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,627 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,631 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:50:50,635 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:50,647 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:50:50,648 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72419012, jitterRate=0.07912737131118774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:50:50,659 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731397850526Initializing all the Stores at 1731397850526Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397850527 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397850528 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397850529 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397850529Cleaning up temporary data from old regions at 1731397850628 (+99 ms)Region opened successfully at 1731397850658 (+30 ms) 2024-11-12T07:50:50,660 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T07:50:50,701 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcccfc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:50:50,738 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T07:50:50,752 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T07:50:50,753 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T07:50:50,756 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T07:50:50,757 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T07:50:50,765 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-12T07:50:50,765 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T07:50:50,797 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T07:50:50,808 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T07:50:50,847 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T07:50:50,850 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T07:50:50,853 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T07:50:50,863 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T07:50:50,867 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T07:50:50,872 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T07:50:50,880 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T07:50:50,882 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T07:50:50,894 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T07:50:50,916 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T07:50:50,951 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T07:50:50,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:50,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:50,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:50,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:50,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:50,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:50,989 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f9fc96254ef8,43613,1731397848605, sessionid=0x1012f501f500000, setting cluster-up flag (Was=false) 2024-11-12T07:50:51,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:51,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:51,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:51,080 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T07:50:51,083 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f9fc96254ef8,43613,1731397848605 2024-11-12T07:50:51,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:51,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:51,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:51,130 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T07:50:51,133 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f9fc96254ef8,43613,1731397848605 2024-11-12T07:50:51,139 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T07:50:51,205 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(746): ClusterId : d6b65e10-50df-47c5-b58b-900b97c93d69 2024-11-12T07:50:51,208 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(746): ClusterId : d6b65e10-50df-47c5-b58b-900b97c93d69 2024-11-12T07:50:51,208 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T07:50:51,208 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T07:50:51,235 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T07:50:51,240 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T07:50:51,241 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T07:50:51,252 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T07:50:51,253 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T07:50:51,253 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T07:50:51,262 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T07:50:51,264 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T07:50:51,265 DEBUG [RS:1;f9fc96254ef8:38735 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9fe35f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:50:51,265 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T07:50:51,266 DEBUG [RS:0;f9fc96254ef8:36017 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3edcc7a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:50:51,270 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f9fc96254ef8,43613,1731397848605 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T07:50:51,291 DEBUG [RS:1;f9fc96254ef8:38735 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f9fc96254ef8:38735 2024-11-12T07:50:51,291 DEBUG [RS:0;f9fc96254ef8:36017 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f9fc96254ef8:36017 2024-11-12T07:50:51,293 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:50:51,294 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:50:51,294 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:50:51,295 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:50:51,295 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f9fc96254ef8:0, corePoolSize=10, maxPoolSize=10 2024-11-12T07:50:51,295 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,295 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:50:51,295 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T07:50:51,296 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T07:50:51,296 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,296 DEBUG [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T07:50:51,296 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T07:50:51,296 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T07:50:51,296 DEBUG [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T07:50:51,299 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(2659): reportForDuty to master=f9fc96254ef8,43613,1731397848605 with port=38735, startcode=1731397849887 2024-11-12T07:50:51,300 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(2659): reportForDuty to master=f9fc96254ef8,43613,1731397848605 with port=36017, startcode=1731397849688 2024-11-12T07:50:51,317 DEBUG [RS:0;f9fc96254ef8:36017 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T07:50:51,321 DEBUG [RS:1;f9fc96254ef8:38735 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T07:50:51,335 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:50:51,335 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T07:50:51,342 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:51,342 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731397881342 2024-11-12T07:50:51,345 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T07:50:51,346 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T07:50:51,343 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T07:50:51,351 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T07:50:51,352 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T07:50:51,353 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T07:50:51,353 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T07:50:51,363 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,371 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T07:50:51,372 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T07:50:51,373 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T07:50:51,379 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T07:50:51,380 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T07:50:51,394 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397851381,5,FailOnTimeoutGroup] 2024-11-12T07:50:51,398 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397851394,5,FailOnTimeoutGroup] 2024-11-12T07:50:51,398 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,400 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T07:50:51,403 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,404 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,406 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T07:50:51,407 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 2024-11-12T07:50:51,424 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52627, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T07:50:51,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33219, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T07:50:51,433 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43613 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:51,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:51,436 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43613 {}] master.ServerManager(517): Registering regionserver=f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:51,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T07:50:51,449 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43613 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:51,450 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43613 {}] master.ServerManager(517): Registering regionserver=f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:51,454 DEBUG [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 2024-11-12T07:50:51,454 DEBUG [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 2024-11-12T07:50:51,454 DEBUG [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T07:50:51,454 DEBUG [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T07:50:51,454 DEBUG [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T07:50:51,454 DEBUG [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T07:50:51,456 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T07:50:51,456 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:51,456 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:51,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T07:50:51,463 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T07:50:51,464 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:51,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:51,465 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T07:50:51,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T07:50:51,472 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:51,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:51,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T07:50:51,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T07:50:51,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:51,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:51,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T07:50:51,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740 2024-11-12T07:50:51,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740 2024-11-12T07:50:51,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T07:50:51,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T07:50:51,487 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:50:51,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T07:50:51,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:50:51,496 DEBUG [RS:1;f9fc96254ef8:38735 {}] zookeeper.ZKUtil(111): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:51,496 DEBUG [RS:0;f9fc96254ef8:36017 {}] zookeeper.ZKUtil(111): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:51,496 WARN [RS:0;f9fc96254ef8:36017 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:50:51,496 WARN [RS:1;f9fc96254ef8:38735 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:50:51,496 INFO [RS:1;f9fc96254ef8:38735 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:50:51,496 INFO [RS:0;f9fc96254ef8:36017 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:50:51,497 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:50:51,497 DEBUG [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:51,497 DEBUG [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:51,497 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67618016, jitterRate=0.007586956024169922}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:50:51,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731397851434Initializing all the Stores at 1731397851434Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397851434Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397851439 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397851439Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397851439Cleaning up temporary data from old regions at 1731397851487 (+48 ms)Region opened successfully at 1731397851501 (+14 ms) 2024-11-12T07:50:51,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T07:50:51,501 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T07:50:51,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T07:50:51,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T07:50:51,502 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T07:50:51,503 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T07:50:51,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731397851501Disabling compacts and flushes for region at 1731397851501Disabling writes for close at 1731397851502 (+1 ms)Writing region close event to WAL at 1731397851503 (+1 ms)Closed at 1731397851503 2024-11-12T07:50:51,505 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:50:51,505 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T07:50:51,511 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f9fc96254ef8,36017,1731397849688] 2024-11-12T07:50:51,511 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f9fc96254ef8,38735,1731397849887] 2024-11-12T07:50:51,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T07:50:51,527 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T07:50:51,529 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T07:50:51,534 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T07:50:51,535 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T07:50:51,556 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T07:50:51,556 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T07:50:51,561 INFO [RS:0;f9fc96254ef8:36017 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T07:50:51,561 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,564 INFO [RS:1;f9fc96254ef8:38735 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T07:50:51,564 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,565 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T07:50:51,565 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T07:50:51,571 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T07:50:51,571 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T07:50:51,572 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,572 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,573 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,573 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,573 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,573 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,573 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,573 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,574 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,574 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,574 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,574 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,574 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:50:51,574 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:50:51,574 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,574 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,574 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,574 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,575 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,575 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,575 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,575 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,575 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,575 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,575 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,575 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:51,576 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:50:51,576 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:50:51,576 DEBUG [RS:1;f9fc96254ef8:38735 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:50:51,576 DEBUG [RS:0;f9fc96254ef8:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:50:51,583 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,583 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,583 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,583 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,584 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,584 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,584 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,584 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,584 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,584 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,38735,1731397849887-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:50:51,584 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,584 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,36017,1731397849688-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:50:51,609 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T07:50:51,609 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T07:50:51,611 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,36017,1731397849688-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,611 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,38735,1731397849887-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,612 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,612 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,612 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.Replication(171): f9fc96254ef8,36017,1731397849688 started 2024-11-12T07:50:51,612 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.Replication(171): f9fc96254ef8,38735,1731397849887 started 2024-11-12T07:50:51,635 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,635 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:51,635 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(1482): Serving as f9fc96254ef8,36017,1731397849688, RpcServer on f9fc96254ef8/172.17.0.2:36017, sessionid=0x1012f501f500001 2024-11-12T07:50:51,635 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(1482): Serving as f9fc96254ef8,38735,1731397849887, RpcServer on f9fc96254ef8/172.17.0.2:38735, sessionid=0x1012f501f500002 2024-11-12T07:50:51,636 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T07:50:51,636 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T07:50:51,636 DEBUG [RS:0;f9fc96254ef8:36017 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:51,636 DEBUG [RS:1;f9fc96254ef8:38735 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:51,637 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,38735,1731397849887' 2024-11-12T07:50:51,637 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,36017,1731397849688' 2024-11-12T07:50:51,637 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T07:50:51,637 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T07:50:51,638 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T07:50:51,638 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T07:50:51,639 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T07:50:51,639 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T07:50:51,640 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T07:50:51,640 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T07:50:51,640 DEBUG [RS:1;f9fc96254ef8:38735 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:51,640 DEBUG [RS:0;f9fc96254ef8:36017 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:51,640 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,36017,1731397849688' 2024-11-12T07:50:51,640 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,38735,1731397849887' 2024-11-12T07:50:51,640 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T07:50:51,640 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T07:50:51,641 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T07:50:51,641 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T07:50:51,642 DEBUG [RS:0;f9fc96254ef8:36017 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T07:50:51,642 INFO [RS:0;f9fc96254ef8:36017 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T07:50:51,642 INFO [RS:0;f9fc96254ef8:36017 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T07:50:51,642 DEBUG [RS:1;f9fc96254ef8:38735 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T07:50:51,643 INFO [RS:1;f9fc96254ef8:38735 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T07:50:51,643 INFO [RS:1;f9fc96254ef8:38735 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T07:50:51,680 WARN [f9fc96254ef8:43613 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:50:51,748 INFO [RS:1;f9fc96254ef8:38735 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:50:51,748 INFO [RS:0;f9fc96254ef8:36017 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:50:51,750 INFO [RS:0;f9fc96254ef8:36017 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f9fc96254ef8%2C36017%2C1731397849688, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,36017,1731397849688, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs, maxLogs=55 2024-11-12T07:50:51,750 INFO [RS:1;f9fc96254ef8:38735 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f9fc96254ef8%2C38735%2C1731397849887, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,38735,1731397849887, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs, maxLogs=55 2024-11-12T07:50:51,786 INFO [RS:0;f9fc96254ef8:36017 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,36017,1731397849688/f9fc96254ef8%2C36017%2C1731397849688.1731397851753 2024-11-12T07:50:51,788 DEBUG [RS:0;f9fc96254ef8:36017 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T07:50:51,796 INFO [RS:1;f9fc96254ef8:38735 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,38735,1731397849887/f9fc96254ef8%2C38735%2C1731397849887.1731397851753 2024-11-12T07:50:51,798 DEBUG [RS:1;f9fc96254ef8:38735 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T07:50:51,933 DEBUG [f9fc96254ef8:43613 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-12T07:50:51,943 DEBUG [f9fc96254ef8:43613 {}] balancer.BalancerClusterState(204): Hosts are {f9fc96254ef8=0} racks are {/default-rack=0} 2024-11-12T07:50:51,946 DEBUG [f9fc96254ef8:43613 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T07:50:51,946 DEBUG [f9fc96254ef8:43613 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T07:50:51,946 DEBUG [f9fc96254ef8:43613 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T07:50:51,946 DEBUG [f9fc96254ef8:43613 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T07:50:51,946 INFO [f9fc96254ef8:43613 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T07:50:51,946 INFO [f9fc96254ef8:43613 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T07:50:51,947 DEBUG [f9fc96254ef8:43613 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T07:50:51,954 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:51,960 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f9fc96254ef8,38735,1731397849887, state=OPENING 2024-11-12T07:50:51,993 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T07:50:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:52,006 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:52,006 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:52,006 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:52,007 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T07:50:52,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f9fc96254ef8,38735,1731397849887}] 2024-11-12T07:50:52,181 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T07:50:52,185 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35633, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T07:50:52,193 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T07:50:52,193 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:50:52,194 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T07:50:52,195 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f9fc96254ef8%2C38735%2C1731397849887.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,38735,1731397849887, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs, maxLogs=55 2024-11-12T07:50:52,215 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,38735,1731397849887/f9fc96254ef8%2C38735%2C1731397849887.meta.1731397852198.meta 2024-11-12T07:50:52,215 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T07:50:52,215 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:52,217 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T07:50:52,220 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T07:50:52,224 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T07:50:52,228 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T07:50:52,229 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:52,229 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T07:50:52,229 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T07:50:52,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T07:50:52,232 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T07:50:52,232 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:52,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:52,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T07:50:52,233 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T07:50:52,234 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:52,234 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:52,234 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T07:50:52,235 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T07:50:52,235 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:52,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:52,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T07:50:52,236 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T07:50:52,236 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:52,236 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:52,236 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T07:50:52,237 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740 2024-11-12T07:50:52,238 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740 2024-11-12T07:50:52,238 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T07:50:52,238 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T07:50:52,239 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:50:52,240 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T07:50:52,240 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67976025, jitterRate=0.012921705842018127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:50:52,241 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T07:50:52,242 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731397852230Writing region info on filesystem at 1731397852230Initializing all the Stores at 1731397852230Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397852230Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397852231 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397852231Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397852231Cleaning up temporary data from old regions at 1731397852238 (+7 ms)Running coprocessor post-open hooks at 1731397852241 (+3 ms)Region opened successfully at 1731397852242 (+1 ms) 2024-11-12T07:50:52,246 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731397852174 2024-11-12T07:50:52,257 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T07:50:52,258 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T07:50:52,258 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:52,260 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f9fc96254ef8,38735,1731397849887, state=OPEN 2024-11-12T07:50:52,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:52,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:52,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:52,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:52,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:52,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:52,281 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:52,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T07:50:52,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f9fc96254ef8,38735,1731397849887 in 272 msec 2024-11-12T07:50:52,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T07:50:52,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 773 msec 2024-11-12T07:50:52,293 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:50:52,293 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T07:50:52,322 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:52,324 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,38735,1731397849887, seqNum=-1] 2024-11-12T07:50:52,347 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:52,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40789, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:52,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1880 sec 2024-11-12T07:50:52,369 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731397852369, completionTime=-1 2024-11-12T07:50:52,372 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-12T07:50:52,373 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T07:50:52,402 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T07:50:52,402 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731397912402 2024-11-12T07:50:52,402 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731397972402 2024-11-12T07:50:52,402 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-12T07:50:52,405 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-12T07:50:52,415 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43613,1731397848605-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:52,416 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43613,1731397848605-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:52,416 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43613,1731397848605-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:52,417 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f9fc96254ef8:43613, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:52,418 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:52,418 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:52,424 DEBUG [master/f9fc96254ef8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T07:50:52,459 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.367sec 2024-11-12T07:50:52,461 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T07:50:52,462 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T07:50:52,463 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T07:50:52,463 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T07:50:52,463 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T07:50:52,464 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43613,1731397848605-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:50:52,465 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43613,1731397848605-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T07:50:52,466 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T07:50:52,467 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T07:50:52,468 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43613,1731397848605-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:52,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:52,508 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T07:50:52,508 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T07:50:52,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43613,-1 for getting cluster id 2024-11-12T07:50:52,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:52,525 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd6b65e10-50df-47c5-b58b-900b97c93d69' 2024-11-12T07:50:52,529 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:52,529 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d6b65e10-50df-47c5-b58b-900b97c93d69" 2024-11-12T07:50:52,530 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@419b79e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:52,530 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43613,-1] 2024-11-12T07:50:52,533 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:52,536 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:52,539 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:52,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:52,543 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:52,552 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,38735,1731397849887, seqNum=-1] 2024-11-12T07:50:52,552 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:52,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:52,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f9fc96254ef8,43613,1731397848605 2024-11-12T07:50:52,583 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T07:50:52,590 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f9fc96254ef8,43613,1731397848605 2024-11-12T07:50:52,606 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d9f3d9 2024-11-12T07:50:52,607 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T07:50:52,611 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T07:50:52,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731397852577', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:50:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731397852577 2024-11-12T07:50:52,633 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731397852577 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:50:52,635 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:52,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731397852577" procId is: 4 2024-11-12T07:50:52,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731397852577 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:50:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:50:52,658 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731397852577-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3e3d9a564615298f90405228aaa9d9d8, NAME => 'testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731397852577', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 2024-11-12T07:50:52,666 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731397852577-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:52,667 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731397852577-pool-0 {}] regionserver.HRegion(1722): Closing 3e3d9a564615298f90405228aaa9d9d8, disabling compactions & flushes 2024-11-12T07:50:52,667 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731397852577-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:52,667 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731397852577-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:52,667 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731397852577-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. after waiting 0 ms 2024-11-12T07:50:52,667 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731397852577-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:52,667 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731397852577-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:52,667 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731397852577-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3e3d9a564615298f90405228aaa9d9d8: Waiting for close lock at 1731397852667Disabling compacts and flushes for region at 1731397852667Disabling writes for close at 1731397852667Writing region close event to WAL at 1731397852667Closed at 1731397852667 2024-11-12T07:50:52,669 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731397852577 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:50:52,673 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731397852669"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397852669"}]},"ts":"1731397852669"} 2024-11-12T07:50:52,678 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:50:52,680 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731397852577 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:50:52,682 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731397852577","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397852680"}]},"ts":"1731397852680"} 2024-11-12T07:50:52,690 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731397852577, state=ENABLING in hbase:meta 2024-11-12T07:50:52,691 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f9fc96254ef8=0} racks are {/default-rack=0} 2024-11-12T07:50:52,692 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T07:50:52,692 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T07:50:52,692 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T07:50:52,692 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T07:50:52,692 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T07:50:52,692 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T07:50:52,692 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T07:50:52,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, ASSIGN}] 2024-11-12T07:50:52,697 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, ASSIGN 2024-11-12T07:50:52,699 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, ASSIGN; state=OFFLINE, location=f9fc96254ef8,38735,1731397849887; forceNewPlan=false, retain=false 2024-11-12T07:50:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:50:52,851 INFO [f9fc96254ef8:43613 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T07:50:52,852 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3e3d9a564615298f90405228aaa9d9d8, regionState=OPENING, regionLocation=f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:52,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, ASSIGN because future has completed 2024-11-12T07:50:52,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e3d9a564615298f90405228aaa9d9d8, server=f9fc96254ef8,38735,1731397849887}] 2024-11-12T07:50:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:50:53,013 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:53,014 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3e3d9a564615298f90405228aaa9d9d8, NAME => 'testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:53,014 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731397852577 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,015 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:53,015 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,015 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,016 INFO [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,021 INFO [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e3d9a564615298f90405228aaa9d9d8 columnFamilyName family 2024-11-12T07:50:53,021 DEBUG [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:53,022 INFO [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] regionserver.HStore(327): Store=3e3d9a564615298f90405228aaa9d9d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:53,022 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,024 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,024 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,025 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,025 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,026 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,033 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:50:53,034 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3e3d9a564615298f90405228aaa9d9d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59309384, jitterRate=-0.11622130870819092}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:50:53,034 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:53,036 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3e3d9a564615298f90405228aaa9d9d8: Running coprocessor pre-open hook at 1731397853015Writing region info on filesystem at 1731397853015Initializing all the Stores at 1731397853016 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397853016Cleaning up temporary data from old regions at 1731397853025 (+9 ms)Running coprocessor post-open hooks at 1731397853035 (+10 ms)Region opened successfully at 1731397853036 (+1 ms) 2024-11-12T07:50:53,037 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8., pid=6, masterSystemTime=1731397853010 2024-11-12T07:50:53,041 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:53,041 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:53,041 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3e3d9a564615298f90405228aaa9d9d8, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:53,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e3d9a564615298f90405228aaa9d9d8, server=f9fc96254ef8,38735,1731397849887 because future has completed 2024-11-12T07:50:53,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T07:50:53,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3e3d9a564615298f90405228aaa9d9d8, server=f9fc96254ef8,38735,1731397849887 in 189 msec 2024-11-12T07:50:53,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T07:50:53,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, ASSIGN in 355 msec 2024-11-12T07:50:53,053 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731397852577 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:50:53,054 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731397852577","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397853053"}]},"ts":"1731397853053"} 2024-11-12T07:50:53,057 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731397852577, state=ENABLED in hbase:meta 2024-11-12T07:50:53,058 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731397852577 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:50:53,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731397852577 in 438 msec 2024-11-12T07:50:53,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:50:53,281 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731397852577 get assigned. Timeout = 60000ms 2024-11-12T07:50:53,281 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731397852577 completed 2024-11-12T07:50:53,283 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:50:53,291 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731397852577 assigned to meta. Checking AM states. 2024-11-12T07:50:53,292 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:50:53,293 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731397852577 assigned. 2024-11-12T07:50:53,304 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731397852577', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8., hostname=f9fc96254ef8,38735,1731397849887, seqNum=2] 2024-11-12T07:50:53,320 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3e3d9a564615298f90405228aaa9d9d8 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-12T07:50:53,349 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 839ca2da0de44000abe06ca47f91375e 2024-11-12T07:50:53,397 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/.tmp/family/839ca2da0de44000abe06ca47f91375e is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-12T07:50:53,403 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/.tmp/family/839ca2da0de44000abe06ca47f91375e 2024-11-12T07:50:53,436 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/.tmp/family/839ca2da0de44000abe06ca47f91375e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e 2024-11-12T07:50:53,437 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e, delay=997 ms 2024-11-12T07:50:53,439 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e, entries=2, sequenceid=6, filesize=4.9 K 2024-11-12T07:50:53,444 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 3e3d9a564615298f90405228aaa9d9d8 in 126ms, sequenceid=6, compaction requested=false 2024-11-12T07:50:53,445 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731397852577' 2024-11-12T07:50:53,447 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3e3d9a564615298f90405228aaa9d9d8: 2024-11-12T07:50:54,436 DEBUG [hfile-prefetch-1731397853437 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 839ca2da0de44000abe06ca47f91375e_0. 2024-11-12T07:50:54,439 DEBUG [hfile-prefetch-1731397853437 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 839ca2da0de44000abe06ca47f91375e_117. 2024-11-12T07:50:54,440 DEBUG [hfile-prefetch-1731397853437 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e, with total blocks 2, and data blocks 1 2024-11-12T07:50:54,441 DEBUG [hfile-prefetch-1731397853437 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-12T07:50:54,441 DEBUG [hfile-prefetch-1731397853437 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-12T07:50:54,441 DEBUG [hfile-prefetch-1731397853437 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e. Will sleep 100ms and try the verification again. 2024-11-12T07:50:54,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=3e3d9a564615298f90405228aaa9d9d8, source=f9fc96254ef8,38735,1731397849887, destination=f9fc96254ef8,36017,1731397849688, warming up region on f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T07:50:54,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=3e3d9a564615298f90405228aaa9d9d8, source=f9fc96254ef8,38735,1731397849887, destination=f9fc96254ef8,36017,1731397849688, running balancer 2024-11-12T07:50:54,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54257, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T07:50:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, REOPEN/MOVE 2024-11-12T07:50:54,460 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, REOPEN/MOVE 2024-11-12T07:50:54,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36017 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36017 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3e3d9a564615298f90405228aaa9d9d8, NAME => 'testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:54,462 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3e3d9a564615298f90405228aaa9d9d8, regionState=CLOSING, regionLocation=f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36017 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:54,463 INFO [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,464 INFO [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e3d9a564615298f90405228aaa9d9d8 columnFamilyName family 2024-11-12T07:50:54,464 DEBUG [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:54,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, REOPEN/MOVE because future has completed 2024-11-12T07:50:54,466 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-12T07:50:54,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3e3d9a564615298f90405228aaa9d9d8, server=f9fc96254ef8,38735,1731397849887}] 2024-11-12T07:50:54,469 DEBUG [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e 2024-11-12T07:50:54,469 INFO [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] regionserver.HStore(327): Store=3e3d9a564615298f90405228aaa9d9d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36017 {}] regionserver.HRegion(1722): Closing 3e3d9a564615298f90405228aaa9d9d8, disabling compactions & flushes 2024-11-12T07:50:54,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36017 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36017 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36017 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. after waiting 0 ms 2024-11-12T07:50:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36017 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,471 DEBUG [StoreFileCloser-3e3d9a564615298f90405228aaa9d9d8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e 2024-11-12T07:50:54,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36017 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36017 {}] regionserver.HRegion(1676): Region close journal for 3e3d9a564615298f90405228aaa9d9d8: Waiting for close lock at 1731397854469Disabling compacts and flushes for region at 1731397854469Disabling writes for close at 1731397854470 (+1 ms)Writing region close event to WAL at 1731397854471 (+1 ms)Closed at 1731397854471 2024-11-12T07:50:54,473 WARN [hfile-prefetch-1731397853437 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-12T07:50:54,474 DEBUG [hfile-prefetch-1731397853437 {}] hfile.PrefetchExecutor(121): Prefetch completed for 839ca2da0de44000abe06ca47f91375e 2024-11-12T07:50:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43613 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-12T07:50:54,528 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T07:50:54,528 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T07:50:54,528 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T07:50:54,528 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T07:50:54,548 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T07:50:54,548 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731397854528 2024-11-12T07:50:54,548 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T07:50:54,625 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,625 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-12T07:50:54,626 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 3e3d9a564615298f90405228aaa9d9d8, disabling compactions & flushes 2024-11-12T07:50:54,626 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,626 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,627 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. after waiting 0 ms 2024-11-12T07:50:54,627 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,628 DEBUG [StoreFileCloser-3e3d9a564615298f90405228aaa9d9d8-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 839ca2da0de44000abe06ca47f91375e, starting offset: 0, end offset: 9223372036854775807 2024-11-12T07:50:54,630 DEBUG [StoreFileCloser-3e3d9a564615298f90405228aaa9d9d8-family-1 {}] bucket.BucketCache(857): removed key 839ca2da0de44000abe06ca47f91375e_0 from back map with offset lock 2094080 in the evict process 2024-11-12T07:50:54,634 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-12T07:50:54,635 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,635 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 3e3d9a564615298f90405228aaa9d9d8: Waiting for close lock at 1731397854626Running coprocessor pre-close hooks at 1731397854626Disabling compacts and flushes for region at 1731397854626Disabling writes for close at 1731397854627 (+1 ms)Writing region close event to WAL at 1731397854630 (+3 ms)Running coprocessor post-close hooks at 1731397854634 (+4 ms)Closed at 1731397854635 (+1 ms) 2024-11-12T07:50:54,636 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 3e3d9a564615298f90405228aaa9d9d8 move to f9fc96254ef8,36017,1731397849688 record at close sequenceid=6 2024-11-12T07:50:54,639 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,640 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3e3d9a564615298f90405228aaa9d9d8, regionState=CLOSED 2024-11-12T07:50:54,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3e3d9a564615298f90405228aaa9d9d8, server=f9fc96254ef8,38735,1731397849887 because future has completed 2024-11-12T07:50:54,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T07:50:54,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 3e3d9a564615298f90405228aaa9d9d8, server=f9fc96254ef8,38735,1731397849887 in 178 msec 2024-11-12T07:50:54,648 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T07:50:54,648 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, REOPEN/MOVE; state=CLOSED, location=f9fc96254ef8,36017,1731397849688; forceNewPlan=false, retain=false 2024-11-12T07:50:54,649 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T07:50:54,649 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T07:50:54,649 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T07:50:54,653 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T07:50:54,654 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731397854649 2024-11-12T07:50:54,654 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T07:50:54,665 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T07:50:54,665 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T07:50:54,670 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T07:50:54,670 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T07:50:54,674 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T07:50:54,674 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731397854665 2024-11-12T07:50:54,674 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T07:50:54,799 INFO [f9fc96254ef8:43613 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T07:50:54,799 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3e3d9a564615298f90405228aaa9d9d8, regionState=OPENING, regionLocation=f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:54,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, REOPEN/MOVE because future has completed 2024-11-12T07:50:54,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e3d9a564615298f90405228aaa9d9d8, server=f9fc96254ef8,36017,1731397849688}] 2024-11-12T07:50:54,962 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,962 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3e3d9a564615298f90405228aaa9d9d8, NAME => 'testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:54,963 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731397852577 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,963 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:54,964 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,964 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,965 INFO [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,967 INFO [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e3d9a564615298f90405228aaa9d9d8 columnFamilyName family 2024-11-12T07:50:54,967 DEBUG [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:54,969 DEBUG [StoreFileOpener-3e3d9a564615298f90405228aaa9d9d8-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e, delay=901 ms 2024-11-12T07:50:54,971 DEBUG [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e 2024-11-12T07:50:54,971 INFO [StoreOpener-3e3d9a564615298f90405228aaa9d9d8-1 {}] regionserver.HStore(327): Store=3e3d9a564615298f90405228aaa9d9d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:54,971 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,972 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,972 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,972 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,972 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,976 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,976 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 3e3d9a564615298f90405228aaa9d9d8; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65405229, jitterRate=-0.025386139750480652}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:50:54,976 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:54,977 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3e3d9a564615298f90405228aaa9d9d8: Running coprocessor pre-open hook at 1731397854964Writing region info on filesystem at 1731397854964Initializing all the Stores at 1731397854965 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397854965Cleaning up temporary data from old regions at 1731397854972 (+7 ms)Running coprocessor post-open hooks at 1731397854977 (+5 ms)Region opened successfully at 1731397854977 2024-11-12T07:50:54,978 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8., pid=9, masterSystemTime=1731397854959 2024-11-12T07:50:54,981 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,981 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:54,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3e3d9a564615298f90405228aaa9d9d8, regionState=OPEN, openSeqNum=10, regionLocation=f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:54,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e3d9a564615298f90405228aaa9d9d8, server=f9fc96254ef8,36017,1731397849688 because future has completed 2024-11-12T07:50:54,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-12T07:50:54,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3e3d9a564615298f90405228aaa9d9d8, server=f9fc96254ef8,36017,1731397849688 in 178 msec 2024-11-12T07:50:54,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731397852577, region=3e3d9a564615298f90405228aaa9d9d8, REOPEN/MOVE in 528 msec 2024-11-12T07:50:55,013 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T07:50:55,013 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:50:55,014 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:50:55,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:55,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:55,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:55,019 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T07:50:55,019 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-12T07:50:55,019 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f9fc96254ef8,43613,1731397848605 2024-11-12T07:50:55,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:55,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:55,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:55,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:55,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:55,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:55,047 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T07:50:55,047 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:50:55,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:55,047 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:50:55,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:55,048 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f9fc96254ef8,36017,1731397849688' ***** 2024-11-12T07:50:55,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:55,048 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T07:50:55,048 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f9fc96254ef8,38735,1731397849887' ***** 2024-11-12T07:50:55,048 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:55,048 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T07:50:55,048 INFO [RS:1;f9fc96254ef8:38735 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T07:50:55,048 INFO [RS:0;f9fc96254ef8:36017 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T07:50:55,049 INFO [RS:0;f9fc96254ef8:36017 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/bucket.persistence 2024-11-12T07:50:55,049 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T07:50:55,049 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T07:50:55,049 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T07:50:55,049 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T07:50:55,049 INFO [RS:1;f9fc96254ef8:38735 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/bucket.persistence 2024-11-12T07:50:55,049 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T07:50:55,049 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T07:50:55,049 INFO [RS:0;f9fc96254ef8:36017 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T07:50:55,049 INFO [RS:1;f9fc96254ef8:38735 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T07:50:55,050 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T07:50:55,050 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T07:50:55,050 DEBUG [RS:0;f9fc96254ef8:36017 {}] bucket.BucketCache(1488): Thread RS:0;f9fc96254ef8:36017 started persisting bucket cache to file 2024-11-12T07:50:55,050 DEBUG [RS:0;f9fc96254ef8:36017 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T07:50:55,050 DEBUG [RS:0;f9fc96254ef8:36017 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T07:50:55,053 DEBUG [RS:0;f9fc96254ef8:36017 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T07:50:55,053 DEBUG [RS:0;f9fc96254ef8:36017 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731397855050 2024-11-12T07:50:55,053 DEBUG [RS:0;f9fc96254ef8:36017 {}] bucket.BucketCache(1510): Thread RS:0;f9fc96254ef8:36017 finished persisting bucket cache to file, renaming 2024-11-12T07:50:55,053 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T07:50:55,054 INFO [RS:0;f9fc96254ef8:36017 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T07:50:55,054 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T07:50:55,054 INFO [RS:0;f9fc96254ef8:36017 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T07:50:55,054 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:55,055 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(959): stopping server f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:55,055 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:50:55,055 INFO [RS:0;f9fc96254ef8:36017 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f9fc96254ef8:36017. 2024-11-12T07:50:55,055 DEBUG [RS:0;f9fc96254ef8:36017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:50:55,055 DEBUG [RS:0;f9fc96254ef8:36017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:55,055 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T07:50:55,055 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T07:50:55,055 DEBUG [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(1325): Online Regions={3e3d9a564615298f90405228aaa9d9d8=testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8.} 2024-11-12T07:50:55,055 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3e3d9a564615298f90405228aaa9d9d8, disabling compactions & flushes 2024-11-12T07:50:55,055 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:55,056 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:55,056 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. after waiting 0 ms 2024-11-12T07:50:55,056 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:55,056 DEBUG [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(1351): Waiting on 3e3d9a564615298f90405228aaa9d9d8 2024-11-12T07:50:55,056 DEBUG [StoreFileCloser-3e3d9a564615298f90405228aaa9d9d8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/family/839ca2da0de44000abe06ca47f91375e 2024-11-12T07:50:55,060 DEBUG [RS:1;f9fc96254ef8:38735 {}] bucket.BucketCache(1488): Thread RS:1;f9fc96254ef8:38735 started persisting bucket cache to file 2024-11-12T07:50:55,060 DEBUG [RS:1;f9fc96254ef8:38735 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T07:50:55,060 DEBUG [RS:1;f9fc96254ef8:38735 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T07:50:55,062 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnRegionMove1731397852577/3e3d9a564615298f90405228aaa9d9d8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-12T07:50:55,063 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:55,063 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3e3d9a564615298f90405228aaa9d9d8: Waiting for close lock at 1731397855055Running coprocessor pre-close hooks at 1731397855055Disabling compacts and flushes for region at 1731397855055Disabling writes for close at 1731397855056 (+1 ms)Writing region close event to WAL at 1731397855057 (+1 ms)Running coprocessor post-close hooks at 1731397855062 (+5 ms)Closed at 1731397855063 (+1 ms) 2024-11-12T07:50:55,063 DEBUG [RS:1;f9fc96254ef8:38735 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T07:50:55,063 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8. 2024-11-12T07:50:55,063 DEBUG [RS:1;f9fc96254ef8:38735 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731397855060 2024-11-12T07:50:55,063 DEBUG [RS:1;f9fc96254ef8:38735 {}] bucket.BucketCache(1510): Thread RS:1;f9fc96254ef8:38735 finished persisting bucket cache to file, renaming 2024-11-12T07:50:55,063 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T07:50:55,064 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T07:50:55,064 INFO [RS:1;f9fc96254ef8:38735 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T07:50:55,064 INFO [RS:1;f9fc96254ef8:38735 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T07:50:55,064 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(959): stopping server f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:55,064 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:50:55,064 INFO [RS:1;f9fc96254ef8:38735 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f9fc96254ef8:38735. 2024-11-12T07:50:55,064 DEBUG [RS:1;f9fc96254ef8:38735 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:50:55,064 DEBUG [RS:1;f9fc96254ef8:38735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:55,064 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T07:50:55,064 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T07:50:55,065 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T07:50:55,065 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T07:50:55,065 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T07:50:55,065 DEBUG [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T07:50:55,065 DEBUG [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T07:50:55,065 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T07:50:55,065 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T07:50:55,065 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T07:50:55,066 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T07:50:55,066 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T07:50:55,066 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T07:50:55,066 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-12T07:50:55,076 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c35eb22f7be047bc9b11e1e890e3ee48 2024-11-12T07:50:55,089 INFO [regionserver/f9fc96254ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:50:55,090 INFO [regionserver/f9fc96254ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:50:55,092 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/info/c35eb22f7be047bc9b11e1e890e3ee48 is 205, key is testBlockEvictionOnRegionMove1731397852577,,1731397852612.3e3d9a564615298f90405228aaa9d9d8./info:regioninfo/1731397854981/Put/seqid=0 2024-11-12T07:50:55,093 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/info/c35eb22f7be047bc9b11e1e890e3ee48 2024-11-12T07:50:55,103 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6bb25ade3ada40fb9085674ffe5faebf 2024-11-12T07:50:55,116 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/ns/6bb25ade3ada40fb9085674ffe5faebf is 43, key is default/ns:d/1731397852355/Put/seqid=0 2024-11-12T07:50:55,116 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/ns/6bb25ade3ada40fb9085674ffe5faebf 2024-11-12T07:50:55,125 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3f209f3ae1d54d3e9a2d4db0ceb8b082 2024-11-12T07:50:55,140 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/table/3f209f3ae1d54d3e9a2d4db0ceb8b082 is 78, key is testBlockEvictionOnRegionMove1731397852577/table:state/1731397853053/Put/seqid=0 2024-11-12T07:50:55,140 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/table/3f209f3ae1d54d3e9a2d4db0ceb8b082 2024-11-12T07:50:55,144 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/info/c35eb22f7be047bc9b11e1e890e3ee48 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/c35eb22f7be047bc9b11e1e890e3ee48 2024-11-12T07:50:55,146 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/c35eb22f7be047bc9b11e1e890e3ee48, delay=907 ms 2024-11-12T07:50:55,148 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/c35eb22f7be047bc9b11e1e890e3ee48, entries=15, sequenceid=15, filesize=7.6 K 2024-11-12T07:50:55,149 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/ns/6bb25ade3ada40fb9085674ffe5faebf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/6bb25ade3ada40fb9085674ffe5faebf 2024-11-12T07:50:55,151 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/6bb25ade3ada40fb9085674ffe5faebf, delay=962 ms 2024-11-12T07:50:55,152 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/6bb25ade3ada40fb9085674ffe5faebf, entries=2, sequenceid=15, filesize=5.0 K 2024-11-12T07:50:55,153 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/table/3f209f3ae1d54d3e9a2d4db0ceb8b082 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/3f209f3ae1d54d3e9a2d4db0ceb8b082 2024-11-12T07:50:55,155 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/3f209f3ae1d54d3e9a2d4db0ceb8b082, delay=901 ms 2024-11-12T07:50:55,156 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/3f209f3ae1d54d3e9a2d4db0ceb8b082, entries=2, sequenceid=15, filesize=5.3 K 2024-11-12T07:50:55,157 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=15, compaction requested=false 2024-11-12T07:50:55,158 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T07:50:55,159 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/c35eb22f7be047bc9b11e1e890e3ee48 2024-11-12T07:50:55,160 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/6bb25ade3ada40fb9085674ffe5faebf 2024-11-12T07:50:55,160 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/3f209f3ae1d54d3e9a2d4db0ceb8b082 2024-11-12T07:50:55,165 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-12T07:50:55,165 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T07:50:55,166 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T07:50:55,166 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731397855065Running coprocessor pre-close hooks at 1731397855065Disabling compacts and flushes for region at 1731397855065Disabling writes for close at 1731397855066 (+1 ms)Obtaining lock to block concurrent updates at 1731397855066Preparing flush snapshotting stores in 1588230740 at 1731397855066Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731397855067 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731397855067Flushing 1588230740/info: creating writer at 1731397855068 (+1 ms)Flushing 1588230740/info: appending metadata at 1731397855091 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731397855091Flushing 1588230740/ns: creating writer at 1731397855096 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731397855115 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731397855115Flushing 1588230740/table: creating writer at 1731397855119 (+4 ms)Flushing 1588230740/table: appending metadata at 1731397855138 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731397855139 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@244fc453: reopening flushed file at 1731397855144 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c7ab539: reopening flushed file at 1731397855148 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7440d29: reopening flushed file at 1731397855153 (+5 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=15, compaction requested=false at 1731397855157 (+4 ms)Writing region close event to WAL at 1731397855161 (+4 ms)Running coprocessor post-close hooks at 1731397855165 (+4 ms)Closed at 1731397855166 (+1 ms) 2024-11-12T07:50:55,166 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T07:50:55,256 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(976): stopping server f9fc96254ef8,36017,1731397849688; all regions closed. 2024-11-12T07:50:55,260 DEBUG [RS:0;f9fc96254ef8:36017 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs 2024-11-12T07:50:55,261 INFO [RS:0;f9fc96254ef8:36017 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C36017%2C1731397849688:(num 1731397851753) 2024-11-12T07:50:55,261 DEBUG [RS:0;f9fc96254ef8:36017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:55,261 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:50:55,261 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:50:55,261 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.ChoreService(370): Chore service for: regionserver/f9fc96254ef8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T07:50:55,261 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T07:50:55,261 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T07:50:55,261 INFO [regionserver/f9fc96254ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:50:55,261 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T07:50:55,262 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:50:55,262 INFO [RS:0;f9fc96254ef8:36017 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36017 2024-11-12T07:50:55,265 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(976): stopping server f9fc96254ef8,38735,1731397849887; all regions closed. 2024-11-12T07:50:55,268 DEBUG [RS:1;f9fc96254ef8:38735 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs 2024-11-12T07:50:55,268 INFO [RS:1;f9fc96254ef8:38735 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C38735%2C1731397849887.meta:.meta(num 1731397852198) 2024-11-12T07:50:55,270 DEBUG [RS:1;f9fc96254ef8:38735 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs 2024-11-12T07:50:55,270 INFO [RS:1;f9fc96254ef8:38735 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C38735%2C1731397849887:(num 1731397851753) 2024-11-12T07:50:55,270 DEBUG [RS:1;f9fc96254ef8:38735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:55,270 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:50:55,270 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:50:55,271 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.ChoreService(370): Chore service for: regionserver/f9fc96254ef8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T07:50:55,271 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:50:55,271 INFO [regionserver/f9fc96254ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:50:55,271 INFO [RS:1;f9fc96254ef8:38735 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38735 2024-11-12T07:50:55,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:50:55,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f9fc96254ef8,36017,1731397849688 2024-11-12T07:50:55,296 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:50:55,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f9fc96254ef8,38735,1731397849887 2024-11-12T07:50:55,304 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:50:55,313 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f9fc96254ef8,36017,1731397849688] 2024-11-12T07:50:55,329 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f9fc96254ef8,36017,1731397849688 already deleted, retry=false 2024-11-12T07:50:55,329 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f9fc96254ef8,36017,1731397849688 expired; onlineServers=1 2024-11-12T07:50:55,329 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f9fc96254ef8,38735,1731397849887] 2024-11-12T07:50:55,338 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f9fc96254ef8,38735,1731397849887 already deleted, retry=false 2024-11-12T07:50:55,338 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f9fc96254ef8,38735,1731397849887 expired; onlineServers=0 2024-11-12T07:50:55,338 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f9fc96254ef8,43613,1731397848605' ***** 2024-11-12T07:50:55,338 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T07:50:55,338 INFO [M:0;f9fc96254ef8:43613 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:50:55,339 INFO [M:0;f9fc96254ef8:43613 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:50:55,339 DEBUG [M:0;f9fc96254ef8:43613 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T07:50:55,339 DEBUG [M:0;f9fc96254ef8:43613 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T07:50:55,339 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397851381 {}] cleaner.HFileCleaner(306): Exit Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397851381,5,FailOnTimeoutGroup] 2024-11-12T07:50:55,339 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397851394 {}] cleaner.HFileCleaner(306): Exit Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397851394,5,FailOnTimeoutGroup] 2024-11-12T07:50:55,339 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T07:50:55,339 INFO [M:0;f9fc96254ef8:43613 {}] hbase.ChoreService(370): Chore service for: master/f9fc96254ef8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T07:50:55,339 INFO [M:0;f9fc96254ef8:43613 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:50:55,340 DEBUG [M:0;f9fc96254ef8:43613 {}] master.HMaster(1795): Stopping service threads 2024-11-12T07:50:55,340 INFO [M:0;f9fc96254ef8:43613 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T07:50:55,340 INFO [M:0;f9fc96254ef8:43613 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T07:50:55,341 INFO [M:0;f9fc96254ef8:43613 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T07:50:55,341 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T07:50:55,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T07:50:55,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:55,351 DEBUG [M:0;f9fc96254ef8:43613 {}] zookeeper.ZKUtil(347): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T07:50:55,352 WARN [M:0;f9fc96254ef8:43613 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T07:50:55,352 INFO [M:0;f9fc96254ef8:43613 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/.lastflushedseqids 2024-11-12T07:50:55,362 INFO [M:0;f9fc96254ef8:43613 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T07:50:55,363 INFO [M:0;f9fc96254ef8:43613 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T07:50:55,363 DEBUG [M:0;f9fc96254ef8:43613 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T07:50:55,363 INFO [M:0;f9fc96254ef8:43613 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:55,363 DEBUG [M:0;f9fc96254ef8:43613 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:55,363 DEBUG [M:0;f9fc96254ef8:43613 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T07:50:55,363 DEBUG [M:0;f9fc96254ef8:43613 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:55,363 INFO [M:0;f9fc96254ef8:43613 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-12T07:50:55,379 DEBUG [M:0;f9fc96254ef8:43613 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 80722bdfd786418295ff33984cb50f7d 2024-11-12T07:50:55,399 DEBUG [M:0;f9fc96254ef8:43613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/80722bdfd786418295ff33984cb50f7d is 82, key is hbase:meta,,1/info:regioninfo/1731397852258/Put/seqid=0 2024-11-12T07:50:55,400 INFO [M:0;f9fc96254ef8:43613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/80722bdfd786418295ff33984cb50f7d 2024-11-12T07:50:55,414 INFO [RS:0;f9fc96254ef8:36017 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:50:55,414 INFO [RS:0;f9fc96254ef8:36017 {}] regionserver.HRegionServer(1031): Exiting; stopping=f9fc96254ef8,36017,1731397849688; zookeeper connection closed. 2024-11-12T07:50:55,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:55,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x1012f501f500001, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:55,415 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5df894f3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5df894f3 2024-11-12T07:50:55,419 DEBUG [M:0;f9fc96254ef8:43613 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e9a265ea97b64a5c99ae8b0b31102f4e 2024-11-12T07:50:55,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:55,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38735-0x1012f501f500002, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:55,422 INFO [RS:1;f9fc96254ef8:38735 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:50:55,422 INFO [RS:1;f9fc96254ef8:38735 {}] regionserver.HRegionServer(1031): Exiting; stopping=f9fc96254ef8,38735,1731397849887; zookeeper connection closed. 2024-11-12T07:50:55,423 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5377b74 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5377b74 2024-11-12T07:50:55,423 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-12T07:50:55,438 DEBUG [M:0;f9fc96254ef8:43613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e9a265ea97b64a5c99ae8b0b31102f4e is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731397853061/Put/seqid=0 2024-11-12T07:50:55,439 INFO [M:0;f9fc96254ef8:43613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e9a265ea97b64a5c99ae8b0b31102f4e 2024-11-12T07:50:55,458 DEBUG [M:0;f9fc96254ef8:43613 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9bfd8002d05e4988b6521cd32bcaf4fd 2024-11-12T07:50:55,479 DEBUG [M:0;f9fc96254ef8:43613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9bfd8002d05e4988b6521cd32bcaf4fd is 69, key is f9fc96254ef8,36017,1731397849688/rs:state/1731397851450/Put/seqid=0 2024-11-12T07:50:55,480 INFO [M:0;f9fc96254ef8:43613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9bfd8002d05e4988b6521cd32bcaf4fd 2024-11-12T07:50:55,484 DEBUG [M:0;f9fc96254ef8:43613 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/80722bdfd786418295ff33984cb50f7d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/80722bdfd786418295ff33984cb50f7d 2024-11-12T07:50:55,488 INFO [M:0;f9fc96254ef8:43613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/80722bdfd786418295ff33984cb50f7d, entries=8, sequenceid=84, filesize=5.5 K 2024-11-12T07:50:55,489 DEBUG [M:0;f9fc96254ef8:43613 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e9a265ea97b64a5c99ae8b0b31102f4e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e9a265ea97b64a5c99ae8b0b31102f4e 2024-11-12T07:50:55,492 INFO [M:0;f9fc96254ef8:43613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e9a265ea97b64a5c99ae8b0b31102f4e, entries=9, sequenceid=84, filesize=6.5 K 2024-11-12T07:50:55,492 DEBUG [M:0;f9fc96254ef8:43613 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9bfd8002d05e4988b6521cd32bcaf4fd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9bfd8002d05e4988b6521cd32bcaf4fd 2024-11-12T07:50:55,495 INFO [M:0;f9fc96254ef8:43613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9bfd8002d05e4988b6521cd32bcaf4fd, entries=2, sequenceid=84, filesize=5.1 K 2024-11-12T07:50:55,497 INFO [M:0;f9fc96254ef8:43613 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 133ms, sequenceid=84, compaction requested=false 2024-11-12T07:50:55,499 INFO [M:0;f9fc96254ef8:43613 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:55,499 DEBUG [M:0;f9fc96254ef8:43613 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731397855363Disabling compacts and flushes for region at 1731397855363Disabling writes for close at 1731397855363Obtaining lock to block concurrent updates at 1731397855363Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731397855363Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731397855364 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731397855364Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731397855364Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731397855398 (+34 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731397855398Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731397855403 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731397855436 (+33 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731397855437 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731397855442 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731397855477 (+35 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731397855478 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13d25118: reopening flushed file at 1731397855484 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a8d19f4: reopening flushed file at 1731397855488 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aa0f3b9: reopening flushed file at 1731397855492 (+4 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 133ms, sequenceid=84, compaction requested=false at 1731397855497 (+5 ms)Writing region close event to WAL at 1731397855499 (+2 ms)Closed at 1731397855499 2024-11-12T07:50:55,502 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:50:55,503 INFO [M:0;f9fc96254ef8:43613 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T07:50:55,503 INFO [M:0;f9fc96254ef8:43613 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43613 2024-11-12T07:50:55,503 INFO [M:0;f9fc96254ef8:43613 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:50:55,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:55,618 INFO [M:0;f9fc96254ef8:43613 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:50:55,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43613-0x1012f501f500000, quorum=127.0.0.1:56168, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:55,622 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T07:50:55,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T07:50:55,643 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:55,652 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f9fc96254ef8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f9fc96254ef8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731397855146 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731397853437 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f9fc96254ef8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731397855151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731397854969 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f9fc96254ef8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=401 (was 394) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7235 (was 7969) 2024-11-12T07:50:55,658 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=401, ProcessCount=11, AvailableMemoryMB=7235 2024-11-12T07:50:55,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/hadoop-log-dir so I do NOT create it in target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4 2024-11-12T07:50:55,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/hadoop-tmp-dir so I do NOT create it in target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4 2024-11-12T07:50:55,659 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:55,662 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/cluster_6470127b-85d5-dda5-d0e2-6d9c410391e7, deleteOnExit=true 2024-11-12T07:50:55,693 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/cluster_6470127b-85d5-dda5-d0e2-6d9c410391e7/zookeeper_0, clientPort=52259, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/cluster_6470127b-85d5-dda5-d0e2-6d9c410391e7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/cluster_6470127b-85d5-dda5-d0e2-6d9c410391e7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T07:50:55,694 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52259 2024-11-12T07:50:55,705 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 with version=8 2024-11-12T07:50:55,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/hbase-staging 2024-11-12T07:50:55,708 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f9fc96254ef8:0 server-side Connection retries=45 2024-11-12T07:50:55,708 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:55,708 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:55,708 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:50:55,708 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:55,709 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:50:55,709 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T07:50:55,709 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:50:55,710 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36995 2024-11-12T07:50:55,711 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36995 connecting to ZooKeeper ensemble=127.0.0.1:52259 2024-11-12T07:50:55,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369950x0, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:50:55,746 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36995-0x1012f503c910000 connected 2024-11-12T07:50:55,814 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:55,814 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:55,819 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:55,819 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6, hbase.cluster.distributed=false 2024-11-12T07:50:55,822 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:50:55,822 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36995 2024-11-12T07:50:55,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36995 2024-11-12T07:50:55,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36995 2024-11-12T07:50:55,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36995 2024-11-12T07:50:55,824 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36995 2024-11-12T07:50:55,845 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f9fc96254ef8:0 server-side Connection retries=45 2024-11-12T07:50:55,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:55,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:55,845 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:50:55,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:55,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:50:55,845 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T07:50:55,845 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:50:55,847 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44587 2024-11-12T07:50:55,849 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44587 connecting to ZooKeeper ensemble=127.0.0.1:52259 2024-11-12T07:50:55,849 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:55,850 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:55,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:445870x0, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:50:55,862 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44587-0x1012f503c910001 connected 2024-11-12T07:50:55,863 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:55,863 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T07:50:55,864 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.cache 2024-11-12T07:50:55,864 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T07:50:55,864 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T07:50:55,864 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T07:50:55,865 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T07:50:55,865 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T07:50:55,865 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T07:50:55,865 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T07:50:55,866 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.persistence is complete. 2024-11-12T07:50:55,866 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T07:50:55,867 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T07:50:55,869 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:50:55,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44587 2024-11-12T07:50:55,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44587 2024-11-12T07:50:55,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44587 2024-11-12T07:50:55,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44587 2024-11-12T07:50:55,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44587 2024-11-12T07:50:55,886 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f9fc96254ef8:0 server-side Connection retries=45 2024-11-12T07:50:55,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:55,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:55,886 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:50:55,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:55,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:50:55,886 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T07:50:55,886 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:50:55,887 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43371 2024-11-12T07:50:55,889 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43371 connecting to ZooKeeper ensemble=127.0.0.1:52259 2024-11-12T07:50:55,889 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:55,889 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:55,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:433710x0, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:50:55,898 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:433710x0, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:55,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43371-0x1012f503c910002 connected 2024-11-12T07:50:55,898 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T07:50:55,898 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.cache 2024-11-12T07:50:55,899 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T07:50:55,899 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T07:50:55,899 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T07:50:55,899 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T07:50:55,899 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-12T07:50:55,899 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T07:50:55,900 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T07:50:55,900 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T07:50:55,900 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.persistence is complete. 2024-11-12T07:50:55,900 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T07:50:55,902 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:50:55,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43371 2024-11-12T07:50:55,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43371 2024-11-12T07:50:55,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43371 2024-11-12T07:50:55,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43371 2024-11-12T07:50:55,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43371 2024-11-12T07:50:55,922 DEBUG [M:0;f9fc96254ef8:36995 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f9fc96254ef8:36995 2024-11-12T07:50:55,923 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f9fc96254ef8,36995,1731397855707 2024-11-12T07:50:55,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:55,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:55,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:55,931 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f9fc96254ef8,36995,1731397855707 2024-11-12T07:50:55,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:55,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T07:50:55,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T07:50:55,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:55,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:55,940 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T07:50:55,941 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f9fc96254ef8,36995,1731397855707 from backup master directory 2024-11-12T07:50:55,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f9fc96254ef8,36995,1731397855707 2024-11-12T07:50:55,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:55,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:55,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:50:55,947 WARN [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:50:55,947 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f9fc96254ef8,36995,1731397855707 2024-11-12T07:50:55,949 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/hbase.id] with ID: 6aaaf532-1d3a-4490-9d82-7ab20ff75319 2024-11-12T07:50:55,949 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/.tmp/hbase.id 2024-11-12T07:50:55,956 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/hbase.id] 2024-11-12T07:50:55,993 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:55,993 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T07:50:55,997 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 4ms. 2024-11-12T07:50:56,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,016 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:50:56,017 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T07:50:56,018 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:50:56,036 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store 2024-11-12T07:50:56,041 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:56,041 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T07:50:56,041 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:56,041 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:56,041 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T07:50:56,041 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:56,041 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:56,041 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731397856041Disabling compacts and flushes for region at 1731397856041Disabling writes for close at 1731397856041Writing region close event to WAL at 1731397856041Closed at 1731397856041 2024-11-12T07:50:56,042 WARN [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/.initializing 2024-11-12T07:50:56,042 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/WALs/f9fc96254ef8,36995,1731397855707 2024-11-12T07:50:56,047 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:50:56,050 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f9fc96254ef8%2C36995%2C1731397855707, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/WALs/f9fc96254ef8,36995,1731397855707, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/oldWALs, maxLogs=10 2024-11-12T07:50:56,067 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/WALs/f9fc96254ef8,36995,1731397855707/f9fc96254ef8%2C36995%2C1731397855707.1731397856050 2024-11-12T07:50:56,067 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T07:50:56,067 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:56,068 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:56,068 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,068 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,069 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T07:50:56,083 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:56,084 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:56,084 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,094 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T07:50:56,094 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:56,094 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:56,094 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,099 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T07:50:56,099 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:56,099 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:56,100 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T07:50:56,104 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:56,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:56,104 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,105 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,105 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,105 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,105 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,105 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:50:56,106 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:50:56,119 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:50:56,120 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65675812, jitterRate=-0.02135413885116577}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:50:56,121 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731397856068Initializing all the Stores at 1731397856068Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397856068Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397856069 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397856069Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397856069Cleaning up temporary data from old regions at 1731397856105 (+36 ms)Region opened successfully at 1731397856121 (+16 ms) 2024-11-12T07:50:56,127 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T07:50:56,133 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60c30ebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:50:56,139 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T07:50:56,139 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T07:50:56,139 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T07:50:56,139 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T07:50:56,140 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T07:50:56,140 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-12T07:50:56,140 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T07:50:56,145 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T07:50:56,147 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T07:50:56,163 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T07:50:56,164 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T07:50:56,165 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T07:50:56,172 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T07:50:56,173 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T07:50:56,175 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T07:50:56,186 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T07:50:56,187 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T07:50:56,197 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T07:50:56,200 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T07:50:56,211 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T07:50:56,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:56,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:56,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:56,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,223 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f9fc96254ef8,36995,1731397855707, sessionid=0x1012f503c910000, setting cluster-up flag (Was=false) 2024-11-12T07:50:56,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,272 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T07:50:56,275 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f9fc96254ef8,36995,1731397855707 2024-11-12T07:50:56,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,322 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T07:50:56,324 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f9fc96254ef8,36995,1731397855707 2024-11-12T07:50:56,325 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T07:50:56,326 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T07:50:56,326 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T07:50:56,327 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T07:50:56,327 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f9fc96254ef8,36995,1731397855707 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T07:50:56,329 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:50:56,329 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:50:56,330 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:50:56,330 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:50:56,330 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f9fc96254ef8:0, corePoolSize=10, maxPoolSize=10 2024-11-12T07:50:56,330 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,330 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:50:56,330 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,332 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:50:56,332 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T07:50:56,333 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:56,333 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731397886333 2024-11-12T07:50:56,333 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T07:50:56,333 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T07:50:56,333 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T07:50:56,333 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T07:50:56,333 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T07:50:56,334 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T07:50:56,334 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T07:50:56,334 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,334 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T07:50:56,334 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T07:50:56,334 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T07:50:56,335 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T07:50:56,335 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T07:50:56,335 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397856335,5,FailOnTimeoutGroup] 2024-11-12T07:50:56,335 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397856335,5,FailOnTimeoutGroup] 2024-11-12T07:50:56,335 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,335 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T07:50:56,335 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,335 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,361 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T07:50:56,361 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 2024-11-12T07:50:56,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:56,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T07:50:56,378 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T07:50:56,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:56,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:56,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T07:50:56,382 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T07:50:56,382 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:56,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:56,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T07:50:56,391 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T07:50:56,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:56,391 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:56,391 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T07:50:56,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T07:50:56,396 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:56,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:56,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T07:50:56,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740 2024-11-12T07:50:56,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740 2024-11-12T07:50:56,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T07:50:56,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T07:50:56,398 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:50:56,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T07:50:56,409 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:50:56,410 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75403609, jitterRate=0.12360133230686188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:50:56,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731397856370Initializing all the Stores at 1731397856370Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397856370Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397856370Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397856370Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397856371 (+1 ms)Cleaning up temporary data from old regions at 1731397856397 (+26 ms)Region opened successfully at 1731397856411 (+14 ms) 2024-11-12T07:50:56,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T07:50:56,411 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T07:50:56,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T07:50:56,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T07:50:56,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T07:50:56,412 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T07:50:56,412 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731397856411Disabling compacts and flushes for region at 1731397856411Disabling writes for close at 1731397856411Writing region close event to WAL at 1731397856412 (+1 ms)Closed at 1731397856412 2024-11-12T07:50:56,413 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:50:56,413 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T07:50:56,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T07:50:56,414 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(746): ClusterId : 6aaaf532-1d3a-4490-9d82-7ab20ff75319 2024-11-12T07:50:56,414 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T07:50:56,414 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(746): ClusterId : 6aaaf532-1d3a-4490-9d82-7ab20ff75319 2024-11-12T07:50:56,414 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T07:50:56,419 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T07:50:56,420 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T07:50:56,447 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T07:50:56,448 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T07:50:56,448 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T07:50:56,448 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T07:50:56,464 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T07:50:56,464 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T07:50:56,465 DEBUG [RS:1;f9fc96254ef8:43371 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d95ad9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:50:56,465 DEBUG [RS:0;f9fc96254ef8:44587 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67e4b523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:50:56,480 DEBUG [RS:0;f9fc96254ef8:44587 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f9fc96254ef8:44587 2024-11-12T07:50:56,480 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T07:50:56,480 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T07:50:56,480 DEBUG [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T07:50:56,481 DEBUG [RS:1;f9fc96254ef8:43371 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f9fc96254ef8:43371 2024-11-12T07:50:56,481 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T07:50:56,481 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T07:50:56,481 DEBUG [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T07:50:56,481 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(2659): reportForDuty to master=f9fc96254ef8,36995,1731397855707 with port=44587, startcode=1731397855844 2024-11-12T07:50:56,482 DEBUG [RS:0;f9fc96254ef8:44587 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T07:50:56,482 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(2659): reportForDuty to master=f9fc96254ef8,36995,1731397855707 with port=43371, startcode=1731397855885 2024-11-12T07:50:56,482 DEBUG [RS:1;f9fc96254ef8:43371 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T07:50:56,485 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53661, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T07:50:56,486 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36995 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f9fc96254ef8,44587,1731397855844 2024-11-12T07:50:56,486 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36995 {}] master.ServerManager(517): Registering regionserver=f9fc96254ef8,44587,1731397855844 2024-11-12T07:50:56,487 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48535, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T07:50:56,488 DEBUG [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 2024-11-12T07:50:56,488 DEBUG [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T07:50:56,488 DEBUG [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T07:50:56,488 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36995 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:56,488 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36995 {}] master.ServerManager(517): Registering regionserver=f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:56,490 DEBUG [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 2024-11-12T07:50:56,490 DEBUG [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T07:50:56,490 DEBUG [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T07:50:56,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:50:56,530 DEBUG [RS:0;f9fc96254ef8:44587 {}] zookeeper.ZKUtil(111): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f9fc96254ef8,44587,1731397855844 2024-11-12T07:50:56,530 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f9fc96254ef8,43371,1731397855885] 2024-11-12T07:50:56,530 WARN [RS:0;f9fc96254ef8:44587 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:50:56,530 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f9fc96254ef8,44587,1731397855844] 2024-11-12T07:50:56,531 INFO [RS:0;f9fc96254ef8:44587 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:50:56,531 DEBUG [RS:1;f9fc96254ef8:43371 {}] zookeeper.ZKUtil(111): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:56,531 DEBUG [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,44587,1731397855844 2024-11-12T07:50:56,531 WARN [RS:1;f9fc96254ef8:43371 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:50:56,531 INFO [RS:1;f9fc96254ef8:43371 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:50:56,531 DEBUG [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:56,536 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T07:50:56,540 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T07:50:56,542 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T07:50:56,548 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T07:50:56,551 INFO [RS:0;f9fc96254ef8:44587 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T07:50:56,551 INFO [RS:1;f9fc96254ef8:43371 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T07:50:56,551 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,551 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,551 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T07:50:56,552 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T07:50:56,552 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T07:50:56,552 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,552 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T07:50:56,552 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,552 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:50:56,553 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:50:56,553 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,553 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,554 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,554 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,554 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,554 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,554 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:50:56,554 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,554 DEBUG [RS:0;f9fc96254ef8:44587 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:50:56,554 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,554 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:56,554 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:50:56,554 DEBUG [RS:1;f9fc96254ef8:43371 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:50:56,559 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,44587,1731397855844-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:50:56,559 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43371,1731397855885-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:50:56,571 WARN [f9fc96254ef8:36995 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:50:56,581 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T07:50:56,581 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43371,1731397855885-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,581 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,581 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.Replication(171): f9fc96254ef8,43371,1731397855885 started 2024-11-12T07:50:56,582 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T07:50:56,583 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,44587,1731397855844-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,583 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,583 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.Replication(171): f9fc96254ef8,44587,1731397855844 started 2024-11-12T07:50:56,601 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,602 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(1482): Serving as f9fc96254ef8,43371,1731397855885, RpcServer on f9fc96254ef8/172.17.0.2:43371, sessionid=0x1012f503c910002 2024-11-12T07:50:56,602 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T07:50:56,602 DEBUG [RS:1;f9fc96254ef8:43371 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:56,602 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,43371,1731397855885' 2024-11-12T07:50:56,602 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T07:50:56,603 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T07:50:56,603 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T07:50:56,603 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T07:50:56,603 DEBUG [RS:1;f9fc96254ef8:43371 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:56,603 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,43371,1731397855885' 2024-11-12T07:50:56,603 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T07:50:56,604 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:56,604 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(1482): Serving as f9fc96254ef8,44587,1731397855844, RpcServer on f9fc96254ef8/172.17.0.2:44587, sessionid=0x1012f503c910001 2024-11-12T07:50:56,604 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T07:50:56,604 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T07:50:56,604 DEBUG [RS:0;f9fc96254ef8:44587 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f9fc96254ef8,44587,1731397855844 2024-11-12T07:50:56,604 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,44587,1731397855844' 2024-11-12T07:50:56,604 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T07:50:56,605 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T07:50:56,605 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T07:50:56,605 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T07:50:56,605 DEBUG [RS:0;f9fc96254ef8:44587 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f9fc96254ef8,44587,1731397855844 2024-11-12T07:50:56,605 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,44587,1731397855844' 2024-11-12T07:50:56,605 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T07:50:56,606 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T07:50:56,606 DEBUG [RS:0;f9fc96254ef8:44587 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T07:50:56,606 DEBUG [RS:1;f9fc96254ef8:43371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T07:50:56,606 INFO [RS:0;f9fc96254ef8:44587 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T07:50:56,606 INFO [RS:1;f9fc96254ef8:43371 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T07:50:56,606 INFO [RS:0;f9fc96254ef8:44587 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T07:50:56,606 INFO [RS:1;f9fc96254ef8:43371 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T07:50:56,707 INFO [RS:1;f9fc96254ef8:43371 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:50:56,707 INFO [RS:0;f9fc96254ef8:44587 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:50:56,708 INFO [RS:1;f9fc96254ef8:43371 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f9fc96254ef8%2C43371%2C1731397855885, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,43371,1731397855885, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs, maxLogs=55 2024-11-12T07:50:56,708 INFO [RS:0;f9fc96254ef8:44587 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f9fc96254ef8%2C44587%2C1731397855844, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,44587,1731397855844, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs, maxLogs=55 2024-11-12T07:50:56,735 INFO [RS:1;f9fc96254ef8:43371 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,43371,1731397855885/f9fc96254ef8%2C43371%2C1731397855885.1731397856711 2024-11-12T07:50:56,735 DEBUG [RS:1;f9fc96254ef8:43371 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T07:50:56,739 INFO [RS:0;f9fc96254ef8:44587 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,44587,1731397855844/f9fc96254ef8%2C44587%2C1731397855844.1731397856711 2024-11-12T07:50:56,740 DEBUG [RS:0;f9fc96254ef8:44587 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T07:50:56,821 DEBUG [f9fc96254ef8:36995 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-12T07:50:56,821 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(204): Hosts are {f9fc96254ef8=0} racks are {/default-rack=0} 2024-11-12T07:50:56,822 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T07:50:56,822 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T07:50:56,822 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T07:50:56,822 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T07:50:56,822 INFO [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T07:50:56,822 INFO [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T07:50:56,822 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T07:50:56,823 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:56,824 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f9fc96254ef8,43371,1731397855885, state=OPENING 2024-11-12T07:50:56,853 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T07:50:56,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:56,864 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T07:50:56,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:56,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:56,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f9fc96254ef8,43371,1731397855885}] 2024-11-12T07:50:56,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:57,018 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T07:50:57,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46815, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T07:50:57,023 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T07:50:57,023 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:50:57,024 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T07:50:57,025 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f9fc96254ef8%2C43371%2C1731397855885.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,43371,1731397855885, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs, maxLogs=55 2024-11-12T07:50:57,045 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,43371,1731397855885/f9fc96254ef8%2C43371%2C1731397855885.meta.1731397857026.meta 2024-11-12T07:50:57,046 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T07:50:57,047 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:57,047 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T07:50:57,047 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T07:50:57,048 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T07:50:57,048 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T07:50:57,048 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:57,048 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T07:50:57,048 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T07:50:57,049 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T07:50:57,050 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T07:50:57,050 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:57,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:57,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T07:50:57,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T07:50:57,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:57,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:57,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T07:50:57,052 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T07:50:57,052 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:57,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:57,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T07:50:57,053 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T07:50:57,053 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:57,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:57,054 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T07:50:57,055 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740 2024-11-12T07:50:57,056 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740 2024-11-12T07:50:57,056 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T07:50:57,056 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T07:50:57,057 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:50:57,058 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T07:50:57,059 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64535097, jitterRate=-0.0383521169424057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:50:57,059 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T07:50:57,060 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731397857049Writing region info on filesystem at 1731397857049Initializing all the Stores at 1731397857049Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397857049Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397857049Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397857049Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397857049Cleaning up temporary data from old regions at 1731397857056 (+7 ms)Running coprocessor post-open hooks at 1731397857059 (+3 ms)Region opened successfully at 1731397857060 (+1 ms) 2024-11-12T07:50:57,061 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731397857017 2024-11-12T07:50:57,064 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T07:50:57,064 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T07:50:57,065 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:57,065 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f9fc96254ef8,43371,1731397855885, state=OPEN 2024-11-12T07:50:57,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:57,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:57,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:57,089 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:57,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:57,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:57,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:57,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T07:50:57,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f9fc96254ef8,43371,1731397855885 in 225 msec 2024-11-12T07:50:57,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T07:50:57,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 679 msec 2024-11-12T07:50:57,095 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:50:57,095 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T07:50:57,098 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:57,098 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,43371,1731397855885, seqNum=-1] 2024-11-12T07:50:57,099 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:57,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54075, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:57,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 784 msec 2024-11-12T07:50:57,111 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731397857111, completionTime=-1 2024-11-12T07:50:57,111 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-12T07:50:57,111 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T07:50:57,115 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T07:50:57,115 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731397917115 2024-11-12T07:50:57,115 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731397977115 2024-11-12T07:50:57,116 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-12T07:50:57,116 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,36995,1731397855707-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:57,116 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,36995,1731397855707-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:57,117 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,36995,1731397855707-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:57,117 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f9fc96254ef8:36995, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:57,117 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:57,119 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:57,120 DEBUG [master/f9fc96254ef8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T07:50:57,126 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.177sec 2024-11-12T07:50:57,126 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T07:50:57,126 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T07:50:57,126 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T07:50:57,126 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T07:50:57,126 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T07:50:57,126 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,36995,1731397855707-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:50:57,127 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,36995,1731397855707-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T07:50:57,128 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T07:50:57,129 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T07:50:57,129 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,36995,1731397855707-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:57,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3076c0d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:57,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,36995,-1 for getting cluster id 2024-11-12T07:50:57,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:57,215 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6aaaf532-1d3a-4490-9d82-7ab20ff75319' 2024-11-12T07:50:57,216 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:57,216 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6aaaf532-1d3a-4490-9d82-7ab20ff75319" 2024-11-12T07:50:57,217 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bacd79d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:57,217 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,36995,-1] 2024-11-12T07:50:57,217 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:57,218 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:57,220 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:57,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:57,223 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:57,226 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,43371,1731397855885, seqNum=-1] 2024-11-12T07:50:57,226 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:57,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:57,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f9fc96254ef8,36995,1731397855707 2024-11-12T07:50:57,232 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T07:50:57,234 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f9fc96254ef8,36995,1731397855707 2024-11-12T07:50:57,234 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4be3ffba 2024-11-12T07:50:57,234 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T07:50:57,236 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T07:50:57,238 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36995 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731397857232', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:50:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36995 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731397857232 2024-11-12T07:50:57,241 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731397857232 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:50:57,241 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:57,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36995 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731397857232" procId is: 4 2024-11-12T07:50:57,242 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731397857232 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:50:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:50:57,254 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731397857232-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 032f5ad5f0f56ef40c04de5915d3a5f8, NAME => 'testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731397857232', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 2024-11-12T07:50:57,262 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731397857232-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:57,266 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731397857232-pool-0 {}] regionserver.HRegion(1722): Closing 032f5ad5f0f56ef40c04de5915d3a5f8, disabling compactions & flushes 2024-11-12T07:50:57,266 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731397857232-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:57,266 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731397857232-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:57,266 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731397857232-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. after waiting 0 ms 2024-11-12T07:50:57,266 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731397857232-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:57,266 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731397857232-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:57,266 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731397857232-pool-0 {}] regionserver.HRegion(1676): Region close journal for 032f5ad5f0f56ef40c04de5915d3a5f8: Waiting for close lock at 1731397857266Disabling compacts and flushes for region at 1731397857266Disabling writes for close at 1731397857266Writing region close event to WAL at 1731397857266Closed at 1731397857266 2024-11-12T07:50:57,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731397857232 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:50:57,269 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731397857268"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397857268"}]},"ts":"1731397857268"} 2024-11-12T07:50:57,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:50:57,273 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731397857232 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:50:57,274 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731397857232","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397857273"}]},"ts":"1731397857273"} 2024-11-12T07:50:57,277 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731397857232, state=ENABLING in hbase:meta 2024-11-12T07:50:57,277 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f9fc96254ef8=0} racks are {/default-rack=0} 2024-11-12T07:50:57,278 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T07:50:57,278 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T07:50:57,278 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T07:50:57,279 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T07:50:57,279 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T07:50:57,279 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T07:50:57,279 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T07:50:57,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731397857232, region=032f5ad5f0f56ef40c04de5915d3a5f8, ASSIGN}] 2024-11-12T07:50:57,281 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731397857232, region=032f5ad5f0f56ef40c04de5915d3a5f8, ASSIGN 2024-11-12T07:50:57,283 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731397857232, region=032f5ad5f0f56ef40c04de5915d3a5f8, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43371,1731397855885; forceNewPlan=false, retain=false 2024-11-12T07:50:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:50:57,434 INFO [f9fc96254ef8:36995 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T07:50:57,435 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=032f5ad5f0f56ef40c04de5915d3a5f8, regionState=OPENING, regionLocation=f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:57,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731397857232, region=032f5ad5f0f56ef40c04de5915d3a5f8, ASSIGN because future has completed 2024-11-12T07:50:57,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 032f5ad5f0f56ef40c04de5915d3a5f8, server=f9fc96254ef8,43371,1731397855885}] 2024-11-12T07:50:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:50:57,596 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:57,596 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 032f5ad5f0f56ef40c04de5915d3a5f8, NAME => 'testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:57,597 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731397857232 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,597 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:57,597 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,597 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,598 INFO [StoreOpener-032f5ad5f0f56ef40c04de5915d3a5f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,601 INFO [StoreOpener-032f5ad5f0f56ef40c04de5915d3a5f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 032f5ad5f0f56ef40c04de5915d3a5f8 columnFamilyName family 2024-11-12T07:50:57,602 DEBUG [StoreOpener-032f5ad5f0f56ef40c04de5915d3a5f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:57,602 INFO [StoreOpener-032f5ad5f0f56ef40c04de5915d3a5f8-1 {}] regionserver.HStore(327): Store=032f5ad5f0f56ef40c04de5915d3a5f8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:57,602 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,603 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,603 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,603 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,603 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,604 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,618 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:50:57,619 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 032f5ad5f0f56ef40c04de5915d3a5f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68079906, jitterRate=0.014469653367996216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:50:57,619 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:57,620 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 032f5ad5f0f56ef40c04de5915d3a5f8: Running coprocessor pre-open hook at 1731397857597Writing region info on filesystem at 1731397857597Initializing all the Stores at 1731397857597Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397857597Cleaning up temporary data from old regions at 1731397857603 (+6 ms)Running coprocessor post-open hooks at 1731397857619 (+16 ms)Region opened successfully at 1731397857620 (+1 ms) 2024-11-12T07:50:57,620 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8., pid=6, masterSystemTime=1731397857593 2024-11-12T07:50:57,624 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:57,624 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:57,624 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=032f5ad5f0f56ef40c04de5915d3a5f8, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:57,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 032f5ad5f0f56ef40c04de5915d3a5f8, server=f9fc96254ef8,43371,1731397855885 because future has completed 2024-11-12T07:50:57,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T07:50:57,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 032f5ad5f0f56ef40c04de5915d3a5f8, server=f9fc96254ef8,43371,1731397855885 in 191 msec 2024-11-12T07:50:57,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T07:50:57,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731397857232, region=032f5ad5f0f56ef40c04de5915d3a5f8, ASSIGN in 354 msec 2024-11-12T07:50:57,637 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731397857232 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:50:57,637 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731397857232","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397857637"}]},"ts":"1731397857637"} 2024-11-12T07:50:57,640 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731397857232, state=ENABLED in hbase:meta 2024-11-12T07:50:57,641 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731397857232 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:50:57,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731397857232 in 403 msec 2024-11-12T07:50:57,732 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:50:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:50:57,870 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731397857232 completed 2024-11-12T07:50:57,870 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731397857232 get assigned. Timeout = 60000ms 2024-11-12T07:50:57,870 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:50:57,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731397857232 assigned to meta. Checking AM states. 2024-11-12T07:50:57,875 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:50:57,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731397857232 assigned. 2024-11-12T07:50:57,880 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731397857232', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8., hostname=f9fc96254ef8,43371,1731397855885, seqNum=2] 2024-11-12T07:50:57,887 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 032f5ad5f0f56ef40c04de5915d3a5f8 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-12T07:50:57,898 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3a9add0d00894c72ae17174ae90ed97d 2024-11-12T07:50:57,915 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/.tmp/family/3a9add0d00894c72ae17174ae90ed97d is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-12T07:50:57,917 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/.tmp/family/3a9add0d00894c72ae17174ae90ed97d 2024-11-12T07:50:57,921 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/.tmp/family/3a9add0d00894c72ae17174ae90ed97d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d 2024-11-12T07:50:57,923 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d, delay=963 ms 2024-11-12T07:50:57,924 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d, entries=2, sequenceid=6, filesize=4.9 K 2024-11-12T07:50:57,925 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 032f5ad5f0f56ef40c04de5915d3a5f8 in 38ms, sequenceid=6, compaction requested=false 2024-11-12T07:50:57,925 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731397857232' 2024-11-12T07:50:57,927 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 032f5ad5f0f56ef40c04de5915d3a5f8: 2024-11-12T07:50:58,887 DEBUG [hfile-prefetch-1731397854969 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3a9add0d00894c72ae17174ae90ed97d_0. 2024-11-12T07:50:58,887 DEBUG [hfile-prefetch-1731397854969 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3a9add0d00894c72ae17174ae90ed97d_117. 2024-11-12T07:50:58,888 DEBUG [hfile-prefetch-1731397854969 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d, with total blocks 2, and data blocks 1 2024-11-12T07:50:58,888 DEBUG [hfile-prefetch-1731397854969 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-12T07:50:58,888 DEBUG [hfile-prefetch-1731397854969 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-12T07:50:58,888 DEBUG [hfile-prefetch-1731397854969 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d. Will sleep 100ms and try the verification again. 2024-11-12T07:50:58,927 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;f9fc96254ef8:43371,5,FailOnTimeoutGroup] 2024-11-12T07:50:58,927 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f9fc96254ef8,43371,1731397855885' ***** 2024-11-12T07:50:58,927 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-12T07:50:58,928 INFO [RS:1;f9fc96254ef8:43371 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T07:50:58,933 INFO [RS:1;f9fc96254ef8:43371 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.persistence 2024-11-12T07:50:58,933 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T07:50:58,933 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T07:50:58,933 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T07:50:58,933 INFO [RS:1;f9fc96254ef8:43371 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T07:50:58,933 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T07:50:58,943 DEBUG [RS:1;f9fc96254ef8:43371 {}] bucket.BucketCache(1488): Thread RS:1;f9fc96254ef8:43371 started persisting bucket cache to file 2024-11-12T07:50:58,943 DEBUG [RS:1;f9fc96254ef8:43371 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T07:50:58,944 DEBUG [RS:1;f9fc96254ef8:43371 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T07:50:58,950 DEBUG [RS:1;f9fc96254ef8:43371 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-12T07:50:58,950 DEBUG [RS:1;f9fc96254ef8:43371 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731397858943 2024-11-12T07:50:58,950 DEBUG [RS:1;f9fc96254ef8:43371 {}] bucket.BucketCache(1510): Thread RS:1;f9fc96254ef8:43371 finished persisting bucket cache to file, renaming 2024-11-12T07:50:58,950 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T07:50:58,950 INFO [RS:1;f9fc96254ef8:43371 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T07:50:58,950 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T07:50:58,951 INFO [RS:1;f9fc96254ef8:43371 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T07:50:58,951 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(3091): Received CLOSE for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:50:58,951 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(959): stopping server f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:58,951 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:50:58,951 INFO [RS:1;f9fc96254ef8:43371 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f9fc96254ef8:43371. 2024-11-12T07:50:58,951 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T07:50:58,951 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 032f5ad5f0f56ef40c04de5915d3a5f8, disabling compactions & flushes 2024-11-12T07:50:58,951 DEBUG [RS:1;f9fc96254ef8:43371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:50:58,951 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:58,951 DEBUG [RS:1;f9fc96254ef8:43371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:58,951 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:58,951 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. after waiting 0 ms 2024-11-12T07:50:58,952 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T07:50:58,952 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:58,952 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T07:50:58,952 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T07:50:58,952 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T07:50:58,952 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-12T07:50:58,952 DEBUG [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 032f5ad5f0f56ef40c04de5915d3a5f8=testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8.} 2024-11-12T07:50:58,952 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T07:50:58,952 DEBUG [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(1351): Waiting on 032f5ad5f0f56ef40c04de5915d3a5f8, 1588230740 2024-11-12T07:50:58,952 DEBUG [StoreFileCloser-032f5ad5f0f56ef40c04de5915d3a5f8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d 2024-11-12T07:50:58,952 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T07:50:58,952 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T07:50:58,952 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T07:50:58,952 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T07:50:58,953 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T07:50:58,952 WARN [hfile-prefetch-1731397854969 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-12T07:50:58,953 DEBUG [hfile-prefetch-1731397854969 {}] hfile.PrefetchExecutor(121): Prefetch completed for 3a9add0d00894c72ae17174ae90ed97d 2024-11-12T07:50:58,953 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-12T07:50:58,956 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-12T07:50:58,957 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:58,957 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 032f5ad5f0f56ef40c04de5915d3a5f8: Waiting for close lock at 1731397858951Running coprocessor pre-close hooks at 1731397858951Disabling compacts and flushes for region at 1731397858951Disabling writes for close at 1731397858952 (+1 ms)Writing region close event to WAL at 1731397858952Running coprocessor post-close hooks at 1731397858957 (+5 ms)Closed at 1731397858957 2024-11-12T07:50:58,957 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:50:58,962 INFO [regionserver/f9fc96254ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:50:58,965 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 76a28f0cb66f4eb5943c502390491856 2024-11-12T07:50:58,968 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-12T07:50:58,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-12T07:50:58,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T07:50:58,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T07:50:58,970 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T07:50:58,970 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731397858968 2024-11-12T07:50:58,970 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-12T07:50:58,982 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/info/76a28f0cb66f4eb5943c502390491856 is 209, key is testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8./info:regioninfo/1731397857624/Put/seqid=0 2024-11-12T07:50:58,983 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/info/76a28f0cb66f4eb5943c502390491856 2024-11-12T07:50:58,992 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ff5e7473920c46739c2d1daa66e45796 2024-11-12T07:50:59,003 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/ns/ff5e7473920c46739c2d1daa66e45796 is 43, key is default/ns:d/1731397857101/Put/seqid=0 2024-11-12T07:50:59,004 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/ns/ff5e7473920c46739c2d1daa66e45796 2024-11-12T07:50:59,012 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f74fb648c3644428b419e7da97165b45 2024-11-12T07:50:59,023 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/table/f74fb648c3644428b419e7da97165b45 is 80, key is testBlockEvictionOnGracefulStop1731397857232/table:state/1731397857637/Put/seqid=0 2024-11-12T07:50:59,024 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/table/f74fb648c3644428b419e7da97165b45 2024-11-12T07:50:59,026 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/info/76a28f0cb66f4eb5943c502390491856 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/76a28f0cb66f4eb5943c502390491856 2024-11-12T07:50:59,028 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/76a28f0cb66f4eb5943c502390491856, delay=932 ms 2024-11-12T07:50:59,028 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/76a28f0cb66f4eb5943c502390491856, entries=10, sequenceid=11, filesize=7.0 K 2024-11-12T07:50:59,029 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/ns/ff5e7473920c46739c2d1daa66e45796 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/ff5e7473920c46739c2d1daa66e45796 2024-11-12T07:50:59,030 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/ff5e7473920c46739c2d1daa66e45796, delay=924 ms 2024-11-12T07:50:59,031 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/ff5e7473920c46739c2d1daa66e45796, entries=2, sequenceid=11, filesize=5.0 K 2024-11-12T07:50:59,031 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/table/f74fb648c3644428b419e7da97165b45 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/f74fb648c3644428b419e7da97165b45 2024-11-12T07:50:59,032 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/f74fb648c3644428b419e7da97165b45, delay=923 ms 2024-11-12T07:50:59,033 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/f74fb648c3644428b419e7da97165b45, entries=2, sequenceid=11, filesize=5.3 K 2024-11-12T07:50:59,034 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=11, compaction requested=false 2024-11-12T07:50:59,034 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/76a28f0cb66f4eb5943c502390491856 2024-11-12T07:50:59,035 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/ff5e7473920c46739c2d1daa66e45796 2024-11-12T07:50:59,035 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/f74fb648c3644428b419e7da97165b45 2024-11-12T07:50:59,039 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-12T07:50:59,039 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T07:50:59,039 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T07:50:59,039 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731397858952Running coprocessor pre-close hooks at 1731397858952Disabling compacts and flushes for region at 1731397858952Disabling writes for close at 1731397858952Obtaining lock to block concurrent updates at 1731397858953 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731397858953Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731397858953Flushing stores of hbase:meta,,1.1588230740 at 1731397858953Flushing 1588230740/info: creating writer at 1731397858953Flushing 1588230740/info: appending metadata at 1731397858982 (+29 ms)Flushing 1588230740/info: closing flushed file at 1731397858982Flushing 1588230740/ns: creating writer at 1731397858986 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731397859003 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731397859003Flushing 1588230740/table: creating writer at 1731397859006 (+3 ms)Flushing 1588230740/table: appending metadata at 1731397859023 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731397859023Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76b82ff0: reopening flushed file at 1731397859026 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@db0df53: reopening flushed file at 1731397859029 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e1f8906: reopening flushed file at 1731397859031 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=11, compaction requested=false at 1731397859034 (+3 ms)Writing region close event to WAL at 1731397859036 (+2 ms)Running coprocessor post-close hooks at 1731397859039 (+3 ms)Closed at 1731397859039 2024-11-12T07:50:59,039 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T07:50:59,152 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(976): stopping server f9fc96254ef8,43371,1731397855885; all regions closed. 2024-11-12T07:50:59,154 DEBUG [RS:1;f9fc96254ef8:43371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs 2024-11-12T07:50:59,154 INFO [RS:1;f9fc96254ef8:43371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C43371%2C1731397855885.meta:.meta(num 1731397857026) 2024-11-12T07:50:59,156 DEBUG [RS:1;f9fc96254ef8:43371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs 2024-11-12T07:50:59,156 INFO [RS:1;f9fc96254ef8:43371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C43371%2C1731397855885:(num 1731397856711) 2024-11-12T07:50:59,156 DEBUG [RS:1;f9fc96254ef8:43371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:59,156 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:50:59,156 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:50:59,156 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.ChoreService(370): Chore service for: regionserver/f9fc96254ef8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T07:50:59,156 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:50:59,156 INFO [regionserver/f9fc96254ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:50:59,157 INFO [RS:1;f9fc96254ef8:43371 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43371 2024-11-12T07:50:59,171 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T07:50:59,171 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T07:50:59,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f9fc96254ef8,43371,1731397855885 2024-11-12T07:50:59,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:50:59,188 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:50:59,189 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f9fc96254ef8,43371,1731397855885] 2024-11-12T07:50:59,204 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f9fc96254ef8,43371,1731397855885 already deleted, retry=false 2024-11-12T07:50:59,205 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f9fc96254ef8,43371,1731397855885 on f9fc96254ef8,36995,1731397855707 2024-11-12T07:50:59,209 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f9fc96254ef8,43371,1731397855885, splitWal=true, meta=true 2024-11-12T07:50:59,211 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f9fc96254ef8,43371,1731397855885, splitWal=true, meta=true 2024-11-12T07:50:59,211 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for f9fc96254ef8,43371,1731397855885 (carryingMeta=true) f9fc96254ef8,43371,1731397855885/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@62613846[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-12T07:50:59,212 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure f9fc96254ef8,43371,1731397855885, splitWal=true, meta=true, isMeta: true 2024-11-12T07:50:59,212 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,43371,1731397855885-splitting 2024-11-12T07:50:59,212 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,43371,1731397855885-splitting dir is empty, no logs to split. 2024-11-12T07:50:59,213 INFO [PEWorker-2 {}] master.SplitWALManager(105): f9fc96254ef8,43371,1731397855885 WAL count=0, meta=true 2024-11-12T07:50:59,215 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,43371,1731397855885-splitting dir is empty, no logs to split. 2024-11-12T07:50:59,215 INFO [PEWorker-2 {}] master.SplitWALManager(105): f9fc96254ef8,43371,1731397855885 WAL count=0, meta=true 2024-11-12T07:50:59,215 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if f9fc96254ef8,43371,1731397855885 WAL splitting is done? wals=0, meta=true 2024-11-12T07:50:59,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T07:50:59,217 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T07:50:59,218 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T07:50:59,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:59,296 INFO [RS:1;f9fc96254ef8:43371 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:50:59,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43371-0x1012f503c910002, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:59,297 INFO [RS:1;f9fc96254ef8:43371 {}] regionserver.HRegionServer(1031): Exiting; stopping=f9fc96254ef8,43371,1731397855885; zookeeper connection closed. 2024-11-12T07:50:59,297 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@36c7a823 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@36c7a823 2024-11-12T07:50:59,369 DEBUG [f9fc96254ef8:36995 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T07:50:59,369 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f9fc96254ef8,44587,1731397855844 2024-11-12T07:50:59,370 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f9fc96254ef8,44587,1731397855844, state=OPENING 2024-11-12T07:50:59,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:59,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:59,386 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T07:50:59,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:59,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:59,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f9fc96254ef8,44587,1731397855844}] 2024-11-12T07:50:59,443 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f9fc96254ef8:0 server-side Connection retries=45 2024-11-12T07:50:59,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:59,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:59,443 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:50:59,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:50:59,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:50:59,443 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T07:50:59,444 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:50:59,444 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44929 2024-11-12T07:50:59,446 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44929 connecting to ZooKeeper ensemble=127.0.0.1:52259 2024-11-12T07:50:59,446 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:59,446 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:50:59,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:449290x0, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:50:59,463 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-12T07:50:59,463 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-12T07:50:59,463 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44929-0x1012f503c910003 connected 2024-11-12T07:50:59,465 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T07:50:59,465 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.cache 2024-11-12T07:50:59,465 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-12T07:50:59,465 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-12T07:50:59,465 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-12T07:50:59,466 INFO [Thread-207 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-12T07:50:59,466 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-12T07:50:59,467 INFO [Thread-207 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-12T07:50:59,467 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T07:50:59,467 INFO [Thread-207 {}] bucket.BucketCache(1706): Retrieved 0 of chunks with blockCount = 0. 2024-11-12T07:50:59,468 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T07:50:59,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:50:59,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-12T07:50:59,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44929 2024-11-12T07:50:59,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44929 2024-11-12T07:50:59,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-12T07:50:59,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-12T07:50:59,470 DEBUG [Thread-207 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-12T07:50:59,471 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-12T07:50:59,471 INFO [Thread-207 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 0 2024-11-12T07:50:59,471 INFO [Thread-207 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.persistence is complete. 2024-11-12T07:50:59,472 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(746): ClusterId : 6aaaf532-1d3a-4490-9d82-7ab20ff75319 2024-11-12T07:50:59,472 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T07:50:59,480 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T07:50:59,480 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T07:50:59,489 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T07:50:59,489 DEBUG [RS:2;f9fc96254ef8:44929 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ef5a550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:50:59,500 DEBUG [RS:2;f9fc96254ef8:44929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f9fc96254ef8:44929 2024-11-12T07:50:59,500 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T07:50:59,500 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T07:50:59,500 DEBUG [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T07:50:59,501 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(2659): reportForDuty to master=f9fc96254ef8,36995,1731397855707 with port=44929, startcode=1731397859443 2024-11-12T07:50:59,501 DEBUG [RS:2;f9fc96254ef8:44929 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T07:50:59,503 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46917, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T07:50:59,504 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36995 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f9fc96254ef8,44929,1731397859443 2024-11-12T07:50:59,504 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36995 {}] master.ServerManager(517): Registering regionserver=f9fc96254ef8,44929,1731397859443 2024-11-12T07:50:59,505 DEBUG [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6 2024-11-12T07:50:59,505 DEBUG [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-12T07:50:59,505 DEBUG [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T07:50:59,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:50:59,513 DEBUG [RS:2;f9fc96254ef8:44929 {}] zookeeper.ZKUtil(111): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f9fc96254ef8,44929,1731397859443 2024-11-12T07:50:59,513 WARN [RS:2;f9fc96254ef8:44929 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:50:59,514 INFO [RS:2;f9fc96254ef8:44929 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:50:59,514 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f9fc96254ef8,44929,1731397859443] 2024-11-12T07:50:59,514 DEBUG [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,44929,1731397859443 2024-11-12T07:50:59,519 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T07:50:59,522 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T07:50:59,522 INFO [RS:2;f9fc96254ef8:44929 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T07:50:59,522 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:59,522 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T07:50:59,523 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T07:50:59,523 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:59,523 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,523 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,523 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,523 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,523 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,523 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:50:59,523 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,523 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,524 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,524 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,524 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,524 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:50:59,524 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:50:59,524 DEBUG [RS:2;f9fc96254ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:50:59,524 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:59,524 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:59,524 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:59,524 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:59,525 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:59,525 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,44929,1731397859443-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:50:59,539 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T07:50:59,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48487, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T07:50:59,543 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T07:50:59,543 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:50:59,544 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T07:50:59,546 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f9fc96254ef8%2C44587%2C1731397855844.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,44587,1731397855844, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs, maxLogs=55 2024-11-12T07:50:59,546 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T07:50:59,546 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,44929,1731397859443-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:59,546 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:59,546 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.Replication(171): f9fc96254ef8,44929,1731397859443 started 2024-11-12T07:50:59,565 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,44587,1731397855844/f9fc96254ef8%2C44587%2C1731397855844.meta.1731397859546.meta 2024-11-12T07:50:59,565 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T07:50:59,566 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:59,566 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T07:50:59,566 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T07:50:59,566 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T07:50:59,566 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T07:50:59,566 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:59,566 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T07:50:59,566 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T07:50:59,568 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:50:59,568 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(1482): Serving as f9fc96254ef8,44929,1731397859443, RpcServer on f9fc96254ef8/172.17.0.2:44929, sessionid=0x1012f503c910003 2024-11-12T07:50:59,568 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T07:50:59,568 DEBUG [RS:2;f9fc96254ef8:44929 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f9fc96254ef8,44929,1731397859443 2024-11-12T07:50:59,568 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,44929,1731397859443' 2024-11-12T07:50:59,568 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T07:50:59,569 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T07:50:59,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-12T07:50:59,570 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T07:50:59,570 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T07:50:59,570 DEBUG [RS:2;f9fc96254ef8:44929 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f9fc96254ef8,44929,1731397859443 2024-11-12T07:50:59,570 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,44929,1731397859443' 2024-11-12T07:50:59,570 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T07:50:59,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T07:50:59,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:59,570 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T07:50:59,571 DEBUG [RS:2;f9fc96254ef8:44929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T07:50:59,571 INFO [RS:2;f9fc96254ef8:44929 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T07:50:59,571 INFO [RS:2;f9fc96254ef8:44929 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T07:50:59,572 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/76a28f0cb66f4eb5943c502390491856, delay=984 ms 2024-11-12T07:50:59,573 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/76a28f0cb66f4eb5943c502390491856 2024-11-12T07:50:59,573 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:59,573 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-12T07:50:59,574 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T07:50:59,574 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:59,576 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/ff5e7473920c46739c2d1daa66e45796, delay=980 ms 2024-11-12T07:50:59,578 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/ff5e7473920c46739c2d1daa66e45796 2024-11-12T07:50:59,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:59,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-12T07:50:59,579 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T07:50:59,579 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:59,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:59,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-12T07:50:59,580 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T07:50:59,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:59,582 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/f74fb648c3644428b419e7da97165b45, delay=907 ms 2024-11-12T07:50:59,583 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/f74fb648c3644428b419e7da97165b45 2024-11-12T07:50:59,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:50:59,583 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T07:50:59,584 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740 2024-11-12T07:50:59,584 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740 2024-11-12T07:50:59,585 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T07:50:59,585 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T07:50:59,585 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:50:59,586 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T07:50:59,586 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64509074, jitterRate=-0.0387398898601532}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:50:59,586 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T07:50:59,587 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731397859567Writing region info on filesystem at 1731397859567Initializing all the Stores at 1731397859567Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397859567Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397859569 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397859569Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397859569Cleaning up temporary data from old regions at 1731397859585 (+16 ms)Running coprocessor post-open hooks at 1731397859586 (+1 ms)Region opened successfully at 1731397859587 (+1 ms) 2024-11-12T07:50:59,588 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731397859539 2024-11-12T07:50:59,591 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T07:50:59,591 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T07:50:59,592 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=f9fc96254ef8,44587,1731397855844 2024-11-12T07:50:59,593 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f9fc96254ef8,44587,1731397855844, state=OPEN 2024-11-12T07:50:59,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:59,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:59,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:50:59,621 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f9fc96254ef8,44587,1731397855844 2024-11-12T07:50:59,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:59,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:59,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:50:59,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-12T07:50:59,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f9fc96254ef8,44587,1731397855844 in 235 msec 2024-11-12T07:50:59,632 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): f9fc96254ef8,43371,1731397855885 had 2 regions 2024-11-12T07:50:59,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T07:50:59,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 412 msec 2024-11-12T07:50:59,633 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f9fc96254ef8,43371,1731397855885, splitWal=true, meta=true, isMeta: false 2024-11-12T07:50:59,633 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,43371,1731397855885-splitting dir is empty, no logs to split. 2024-11-12T07:50:59,633 INFO [PEWorker-3 {}] master.SplitWALManager(105): f9fc96254ef8,43371,1731397855885 WAL count=0, meta=false 2024-11-12T07:50:59,635 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,43371,1731397855885-splitting dir is empty, no logs to split. 2024-11-12T07:50:59,635 INFO [PEWorker-3 {}] master.SplitWALManager(105): f9fc96254ef8,43371,1731397855885 WAL count=0, meta=false 2024-11-12T07:50:59,635 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if f9fc96254ef8,43371,1731397855885 WAL splitting is done? wals=0, meta=false 2024-11-12T07:50:59,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731397857232, region=032f5ad5f0f56ef40c04de5915d3a5f8, ASSIGN}] 2024-11-12T07:50:59,639 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731397857232, region=032f5ad5f0f56ef40c04de5915d3a5f8, ASSIGN 2024-11-12T07:50:59,640 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731397857232, region=032f5ad5f0f56ef40c04de5915d3a5f8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T07:50:59,672 INFO [RS:2;f9fc96254ef8:44929 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:50:59,673 INFO [RS:2;f9fc96254ef8:44929 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f9fc96254ef8%2C44929%2C1731397859443, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,44929,1731397859443, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs, maxLogs=55 2024-11-12T07:50:59,697 INFO [RS:2;f9fc96254ef8:44929 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/WALs/f9fc96254ef8,44929,1731397859443/f9fc96254ef8%2C44929%2C1731397859443.1731397859674 2024-11-12T07:50:59,697 DEBUG [RS:2;f9fc96254ef8:44929 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-12T07:50:59,790 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(204): Hosts are {f9fc96254ef8=0} racks are {/default-rack=0} 2024-11-12T07:50:59,791 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T07:50:59,791 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T07:50:59,791 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T07:50:59,791 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T07:50:59,791 INFO [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T07:50:59,791 INFO [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T07:50:59,791 DEBUG [f9fc96254ef8:36995 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T07:50:59,792 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=032f5ad5f0f56ef40c04de5915d3a5f8, regionState=OPENING, regionLocation=f9fc96254ef8,44587,1731397855844 2024-11-12T07:50:59,793 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f9fc96254ef8:43371 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f9fc96254ef8/172.17.0.2:43371 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:59,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,43371,1731397855885, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,43371,1731397855885, seqNum=-1, error=java.net.ConnectException: Call to address=f9fc96254ef8:43371 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f9fc96254ef8/172.17.0.2:43371 2024-11-12T07:50:59,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,43371,1731397855885, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T07:50:59,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,43371,1731397855885, seqNum=-1 from cache 2024-11-12T07:50:59,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address f9fc96254ef8:43371 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f9fc96254ef8/172.17.0.2:43371 2024-11-12T07:50:59,908 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:59,908 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44587,1731397855844, seqNum=-1] 2024-11-12T07:50:59,908 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:59,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48755, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:59,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731397857232, region=032f5ad5f0f56ef40c04de5915d3a5f8, ASSIGN because future has completed 2024-11-12T07:50:59,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 032f5ad5f0f56ef40c04de5915d3a5f8, server=f9fc96254ef8,44587,1731397855844}] 2024-11-12T07:51:00,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T07:51:00,068 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:51:00,068 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:51:00,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:51:00,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:51:00,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:51:00,069 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T07:51:00,069 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=380076799, stopped=false 2024-11-12T07:51:00,069 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f9fc96254ef8,36995,1731397855707 2024-11-12T07:51:00,069 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:51:00,070 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 032f5ad5f0f56ef40c04de5915d3a5f8, NAME => 'testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:51:00,070 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731397857232 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,070 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:51:00,071 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,071 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,072 INFO [StoreOpener-032f5ad5f0f56ef40c04de5915d3a5f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,073 INFO [StoreOpener-032f5ad5f0f56ef40c04de5915d3a5f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 032f5ad5f0f56ef40c04de5915d3a5f8 columnFamilyName family 2024-11-12T07:51:00,073 DEBUG [StoreOpener-032f5ad5f0f56ef40c04de5915d3a5f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:51:00,074 DEBUG [StoreFileOpener-032f5ad5f0f56ef40c04de5915d3a5f8-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d, delay=980 ms 2024-11-12T07:51:00,076 DEBUG [StoreOpener-032f5ad5f0f56ef40c04de5915d3a5f8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d 2024-11-12T07:51:00,076 INFO [StoreOpener-032f5ad5f0f56ef40c04de5915d3a5f8-1 {}] regionserver.HStore(327): Store=032f5ad5f0f56ef40c04de5915d3a5f8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:51:00,076 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,077 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,077 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,077 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,077 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,078 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,079 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 032f5ad5f0f56ef40c04de5915d3a5f8; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61052350, jitterRate=-0.09024909138679504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:51:00,079 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,080 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 032f5ad5f0f56ef40c04de5915d3a5f8: Running coprocessor pre-open hook at 1731397860071Writing region info on filesystem at 1731397860071Initializing all the Stores at 1731397860071Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397860071Cleaning up temporary data from old regions at 1731397860077 (+6 ms)Running coprocessor post-open hooks at 1731397860079 (+2 ms)Region opened successfully at 1731397860080 (+1 ms) 2024-11-12T07:51:00,080 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8., pid=11, masterSystemTime=1731397860066 2024-11-12T07:51:00,083 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:51:00,083 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:51:00,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=032f5ad5f0f56ef40c04de5915d3a5f8, regionState=OPEN, openSeqNum=10, regionLocation=f9fc96254ef8,44587,1731397855844 2024-11-12T07:51:00,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 032f5ad5f0f56ef40c04de5915d3a5f8, server=f9fc96254ef8,44587,1731397855844 because future has completed 2024-11-12T07:51:00,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-12T07:51:00,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 032f5ad5f0f56ef40c04de5915d3a5f8, server=f9fc96254ef8,44587,1731397855844 in 174 msec 2024-11-12T07:51:00,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-12T07:51:00,093 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server f9fc96254ef8,43371,1731397855885 after splitting done 2024-11-12T07:51:00,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731397857232, region=032f5ad5f0f56ef40c04de5915d3a5f8, ASSIGN in 453 msec 2024-11-12T07:51:00,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure f9fc96254ef8,43371,1731397855885, splitWal=true, meta=true in 888 msec 2024-11-12T07:51:00,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T07:51:00,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T07:51:00,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:00,103 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T07:51:00,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T07:51:00,103 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:51:00,103 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:51:00,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:00,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:51:00,104 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f9fc96254ef8,44587,1731397855844' ***** 2024-11-12T07:51:00,104 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T07:51:00,104 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f9fc96254ef8,44929,1731397859443' ***** 2024-11-12T07:51:00,104 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T07:51:00,104 INFO [RS:0;f9fc96254ef8:44587 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T07:51:00,104 INFO [RS:0;f9fc96254ef8:44587 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.persistence 2024-11-12T07:51:00,104 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T07:51:00,104 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T07:51:00,104 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T07:51:00,104 INFO [RS:2;f9fc96254ef8:44929 {}] bucket.BucketCache(1739): Disabling cache 2024-11-12T07:51:00,104 INFO [RS:0;f9fc96254ef8:44587 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T07:51:00,104 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:51:00,104 INFO [RS:2;f9fc96254ef8:44929 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a55ac65d-97c2-6da1-25f8-5486fbcb2bf4/bucket.persistence 2024-11-12T07:51:00,104 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-12T07:51:00,104 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-12T07:51:00,104 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T07:51:00,104 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-12T07:51:00,105 INFO [RS:2;f9fc96254ef8:44929 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-12T07:51:00,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:00,105 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-12T07:51:00,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:51:00,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:51:00,114 DEBUG [RS:0;f9fc96254ef8:44587 {}] bucket.BucketCache(1488): Thread RS:0;f9fc96254ef8:44587 started persisting bucket cache to file 2024-11-12T07:51:00,115 DEBUG [RS:0;f9fc96254ef8:44587 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T07:51:00,115 DEBUG [RS:0;f9fc96254ef8:44587 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T07:51:00,115 DEBUG [RS:2;f9fc96254ef8:44929 {}] bucket.BucketCache(1488): Thread RS:2;f9fc96254ef8:44929 started persisting bucket cache to file 2024-11-12T07:51:00,115 DEBUG [RS:2;f9fc96254ef8:44929 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-12T07:51:00,115 DEBUG [RS:2;f9fc96254ef8:44929 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-12T07:51:00,118 DEBUG [RS:2;f9fc96254ef8:44929 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T07:51:00,118 DEBUG [RS:0;f9fc96254ef8:44587 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-12T07:51:00,118 DEBUG [RS:2;f9fc96254ef8:44929 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731397860115 2024-11-12T07:51:00,118 DEBUG [RS:0;f9fc96254ef8:44587 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731397860114 2024-11-12T07:51:00,118 DEBUG [RS:2;f9fc96254ef8:44929 {}] bucket.BucketCache(1510): Thread RS:2;f9fc96254ef8:44929 finished persisting bucket cache to file, renaming 2024-11-12T07:51:00,118 DEBUG [RS:0;f9fc96254ef8:44587 {}] bucket.BucketCache(1510): Thread RS:0;f9fc96254ef8:44587 finished persisting bucket cache to file, renaming 2024-11-12T07:51:00,118 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T07:51:00,118 INFO [RS:2;f9fc96254ef8:44929 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T07:51:00,118 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T07:51:00,118 INFO [RS:2;f9fc96254ef8:44929 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T07:51:00,118 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T07:51:00,119 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(959): stopping server f9fc96254ef8,44929,1731397859443 2024-11-12T07:51:00,119 INFO [RS:0;f9fc96254ef8:44587 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T07:51:00,119 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:51:00,119 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T07:51:00,119 INFO [RS:2;f9fc96254ef8:44929 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f9fc96254ef8:44929. 2024-11-12T07:51:00,119 INFO [RS:0;f9fc96254ef8:44587 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T07:51:00,119 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(3091): Received CLOSE for 032f5ad5f0f56ef40c04de5915d3a5f8 2024-11-12T07:51:00,119 DEBUG [RS:2;f9fc96254ef8:44929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:51:00,119 DEBUG [RS:2;f9fc96254ef8:44929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:51:00,119 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(976): stopping server f9fc96254ef8,44929,1731397859443; all regions closed. 2024-11-12T07:51:00,119 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(959): stopping server f9fc96254ef8,44587,1731397855844 2024-11-12T07:51:00,120 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:51:00,120 INFO [RS:0;f9fc96254ef8:44587 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f9fc96254ef8:44587. 2024-11-12T07:51:00,120 DEBUG [RS:0;f9fc96254ef8:44587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:51:00,120 DEBUG [RS:0;f9fc96254ef8:44587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:51:00,120 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T07:51:00,120 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T07:51:00,120 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T07:51:00,120 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T07:51:00,121 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T07:51:00,121 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 032f5ad5f0f56ef40c04de5915d3a5f8, disabling compactions & flushes 2024-11-12T07:51:00,121 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:51:00,121 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:51:00,121 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. after waiting 0 ms 2024-11-12T07:51:00,121 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:51:00,121 DEBUG [RS:2;f9fc96254ef8:44929 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs 2024-11-12T07:51:00,121 INFO [RS:2;f9fc96254ef8:44929 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C44929%2C1731397859443:(num 1731397859674) 2024-11-12T07:51:00,121 DEBUG [RS:2;f9fc96254ef8:44929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:51:00,121 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:51:00,121 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:51:00,121 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-12T07:51:00,122 DEBUG [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 032f5ad5f0f56ef40c04de5915d3a5f8=testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8.} 2024-11-12T07:51:00,122 DEBUG [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(1351): Waiting on 032f5ad5f0f56ef40c04de5915d3a5f8, 1588230740 2024-11-12T07:51:00,122 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-12T07:51:00,122 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.ChoreService(370): Chore service for: regionserver/f9fc96254ef8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T07:51:00,122 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T07:51:00,122 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T07:51:00,122 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T07:51:00,122 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T07:51:00,122 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T07:51:00,122 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T07:51:00,122 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T07:51:00,122 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:51:00,122 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T07:51:00,122 INFO [RS:2;f9fc96254ef8:44929 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44929 2024-11-12T07:51:00,122 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-12T07:51:00,123 INFO [regionserver/f9fc96254ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:51:00,126 INFO [regionserver/f9fc96254ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:51:00,126 DEBUG [StoreFileCloser-032f5ad5f0f56ef40c04de5915d3a5f8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/family/3a9add0d00894c72ae17174ae90ed97d 2024-11-12T07:51:00,135 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/default/testBlockEvictionOnGracefulStop1731397857232/032f5ad5f0f56ef40c04de5915d3a5f8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-12T07:51:00,135 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:51:00,135 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 032f5ad5f0f56ef40c04de5915d3a5f8: Waiting for close lock at 1731397860120Running coprocessor pre-close hooks at 1731397860121 (+1 ms)Disabling compacts and flushes for region at 1731397860121Disabling writes for close at 1731397860121Writing region close event to WAL at 1731397860127 (+6 ms)Running coprocessor post-close hooks at 1731397860135 (+8 ms)Closed at 1731397860135 2024-11-12T07:51:00,135 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8. 2024-11-12T07:51:00,138 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ab8954639bbe4238ad0d6ff810c450f4 2024-11-12T07:51:00,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f9fc96254ef8,44929,1731397859443 2024-11-12T07:51:00,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:51:00,143 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:51:00,151 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/info/ab8954639bbe4238ad0d6ff810c450f4 is 209, key is testBlockEvictionOnGracefulStop1731397857232,,1731397857237.032f5ad5f0f56ef40c04de5915d3a5f8./info:regioninfo/1731397860083/Put/seqid=0 2024-11-12T07:51:00,152 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/info/ab8954639bbe4238ad0d6ff810c450f4 2024-11-12T07:51:00,154 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/.tmp/info/ab8954639bbe4238ad0d6ff810c450f4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/ab8954639bbe4238ad0d6ff810c450f4 2024-11-12T07:51:00,154 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f9fc96254ef8,44929,1731397859443] 2024-11-12T07:51:00,156 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/ab8954639bbe4238ad0d6ff810c450f4, delay=934 ms 2024-11-12T07:51:00,157 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/ab8954639bbe4238ad0d6ff810c450f4, entries=8, sequenceid=19, filesize=6.6 K 2024-11-12T07:51:00,158 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=19, compaction requested=false 2024-11-12T07:51:00,158 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/76a28f0cb66f4eb5943c502390491856 2024-11-12T07:51:00,159 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/info/ab8954639bbe4238ad0d6ff810c450f4 2024-11-12T07:51:00,159 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/ns/ff5e7473920c46739c2d1daa66e45796 2024-11-12T07:51:00,160 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/table/f74fb648c3644428b419e7da97165b45 2024-11-12T07:51:00,162 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f9fc96254ef8,44929,1731397859443 already deleted, retry=false 2024-11-12T07:51:00,162 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f9fc96254ef8,44929,1731397859443 expired; onlineServers=1 2024-11-12T07:51:00,163 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-12T07:51:00,164 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T07:51:00,164 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T07:51:00,164 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731397860122Running coprocessor pre-close hooks at 1731397860122Disabling compacts and flushes for region at 1731397860122Disabling writes for close at 1731397860122Obtaining lock to block concurrent updates at 1731397860122Preparing flush snapshotting stores in 1588230740 at 1731397860122Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731397860123 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731397860123Flushing 1588230740/info: creating writer at 1731397860124 (+1 ms)Flushing 1588230740/info: appending metadata at 1731397860151 (+27 ms)Flushing 1588230740/info: closing flushed file at 1731397860151Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e71bc4f: reopening flushed file at 1731397860154 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=19, compaction requested=false at 1731397860158 (+4 ms)Writing region close event to WAL at 1731397860160 (+2 ms)Running coprocessor post-close hooks at 1731397860164 (+4 ms)Closed at 1731397860164 2024-11-12T07:51:00,164 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T07:51:00,164 INFO [regionserver/f9fc96254ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:51:00,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:51:00,254 INFO [RS:2;f9fc96254ef8:44929 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:51:00,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x1012f503c910003, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:51:00,255 INFO [RS:2;f9fc96254ef8:44929 {}] regionserver.HRegionServer(1031): Exiting; stopping=f9fc96254ef8,44929,1731397859443; zookeeper connection closed. 2024-11-12T07:51:00,255 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@441641c6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@441641c6 2024-11-12T07:51:00,322 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(976): stopping server f9fc96254ef8,44587,1731397855844; all regions closed. 2024-11-12T07:51:00,324 DEBUG [RS:0;f9fc96254ef8:44587 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs 2024-11-12T07:51:00,324 INFO [RS:0;f9fc96254ef8:44587 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C44587%2C1731397855844.meta:.meta(num 1731397859546) 2024-11-12T07:51:00,325 DEBUG [RS:0;f9fc96254ef8:44587 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/oldWALs 2024-11-12T07:51:00,326 INFO [RS:0;f9fc96254ef8:44587 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C44587%2C1731397855844:(num 1731397856711) 2024-11-12T07:51:00,326 DEBUG [RS:0;f9fc96254ef8:44587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:51:00,326 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:51:00,326 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:51:00,326 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.ChoreService(370): Chore service for: regionserver/f9fc96254ef8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-12T07:51:00,326 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:51:00,326 INFO [regionserver/f9fc96254ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:51:00,326 INFO [RS:0;f9fc96254ef8:44587 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44587 2024-11-12T07:51:00,354 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:51:00,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:51:00,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f9fc96254ef8,44587,1731397855844 2024-11-12T07:51:00,363 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f9fc96254ef8,44587,1731397855844] 2024-11-12T07:51:00,371 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f9fc96254ef8,44587,1731397855844 already deleted, retry=false 2024-11-12T07:51:00,371 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f9fc96254ef8,44587,1731397855844 expired; onlineServers=0 2024-11-12T07:51:00,371 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f9fc96254ef8,36995,1731397855707' ***** 2024-11-12T07:51:00,371 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T07:51:00,371 INFO [M:0;f9fc96254ef8:36995 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:51:00,371 INFO [M:0;f9fc96254ef8:36995 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:51:00,371 DEBUG [M:0;f9fc96254ef8:36995 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T07:51:00,371 DEBUG [M:0;f9fc96254ef8:36995 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T07:51:00,371 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T07:51:00,371 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397856335 {}] cleaner.HFileCleaner(306): Exit Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397856335,5,FailOnTimeoutGroup] 2024-11-12T07:51:00,371 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397856335 {}] cleaner.HFileCleaner(306): Exit Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397856335,5,FailOnTimeoutGroup] 2024-11-12T07:51:00,371 INFO [M:0;f9fc96254ef8:36995 {}] hbase.ChoreService(370): Chore service for: master/f9fc96254ef8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T07:51:00,372 INFO [M:0;f9fc96254ef8:36995 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:51:00,372 DEBUG [M:0;f9fc96254ef8:36995 {}] master.HMaster(1795): Stopping service threads 2024-11-12T07:51:00,372 INFO [M:0;f9fc96254ef8:36995 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T07:51:00,372 INFO [M:0;f9fc96254ef8:36995 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T07:51:00,372 INFO [M:0;f9fc96254ef8:36995 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T07:51:00,372 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T07:51:00,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T07:51:00,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:00,379 DEBUG [M:0;f9fc96254ef8:36995 {}] zookeeper.ZKUtil(347): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T07:51:00,379 WARN [M:0;f9fc96254ef8:36995 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T07:51:00,380 INFO [M:0;f9fc96254ef8:36995 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/.lastflushedseqids 2024-11-12T07:51:00,384 INFO [M:0;f9fc96254ef8:36995 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T07:51:00,385 INFO [M:0;f9fc96254ef8:36995 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T07:51:00,385 DEBUG [M:0;f9fc96254ef8:36995 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T07:51:00,385 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:51:00,385 DEBUG [M:0;f9fc96254ef8:36995 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:51:00,385 DEBUG [M:0;f9fc96254ef8:36995 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T07:51:00,385 DEBUG [M:0;f9fc96254ef8:36995 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:51:00,385 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.49 KB heapSize=49.88 KB 2024-11-12T07:51:00,393 DEBUG [M:0;f9fc96254ef8:36995 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8da74d4bee55475a87e97608aa7b8b00 2024-11-12T07:51:00,405 DEBUG [M:0;f9fc96254ef8:36995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8da74d4bee55475a87e97608aa7b8b00 is 82, key is hbase:meta,,1/info:regioninfo/1731397859592/Put/seqid=0 2024-11-12T07:51:00,406 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8da74d4bee55475a87e97608aa7b8b00 2024-11-12T07:51:00,412 DEBUG [M:0;f9fc96254ef8:36995 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0014cd95e08e4f9ba6e62c2daea83372 2024-11-12T07:51:00,425 DEBUG [M:0;f9fc96254ef8:36995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0014cd95e08e4f9ba6e62c2daea83372 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731397857642/Put/seqid=0 2024-11-12T07:51:00,425 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.26 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0014cd95e08e4f9ba6e62c2daea83372 2024-11-12T07:51:00,432 DEBUG [M:0;f9fc96254ef8:36995 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 90ad4870b516462bbee75eb21d8e1c81 2024-11-12T07:51:00,444 DEBUG [M:0;f9fc96254ef8:36995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/90ad4870b516462bbee75eb21d8e1c81 is 69, key is f9fc96254ef8,44587,1731397855844/rs:state/1731397856486/Put/seqid=0 2024-11-12T07:51:00,445 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/90ad4870b516462bbee75eb21d8e1c81 2024-11-12T07:51:00,446 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 90ad4870b516462bbee75eb21d8e1c81 2024-11-12T07:51:00,447 DEBUG [M:0;f9fc96254ef8:36995 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8da74d4bee55475a87e97608aa7b8b00 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8da74d4bee55475a87e97608aa7b8b00 2024-11-12T07:51:00,448 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8da74d4bee55475a87e97608aa7b8b00, entries=14, sequenceid=102, filesize=5.9 K 2024-11-12T07:51:00,448 DEBUG [M:0;f9fc96254ef8:36995 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0014cd95e08e4f9ba6e62c2daea83372 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0014cd95e08e4f9ba6e62c2daea83372 2024-11-12T07:51:00,450 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0014cd95e08e4f9ba6e62c2daea83372, entries=11, sequenceid=102, filesize=6.5 K 2024-11-12T07:51:00,450 DEBUG [M:0;f9fc96254ef8:36995 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/90ad4870b516462bbee75eb21d8e1c81 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/90ad4870b516462bbee75eb21d8e1c81 2024-11-12T07:51:00,451 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 90ad4870b516462bbee75eb21d8e1c81 2024-11-12T07:51:00,451 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/982f80ba-1a05-8a64-a5a6-e30dd5d67828/8d9c5671-b27c-1cdf-3b58-f24c3e2d27c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/90ad4870b516462bbee75eb21d8e1c81, entries=3, sequenceid=102, filesize=5.3 K 2024-11-12T07:51:00,452 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=102, compaction requested=false 2024-11-12T07:51:00,453 INFO [M:0;f9fc96254ef8:36995 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:51:00,454 DEBUG [M:0;f9fc96254ef8:36995 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731397860385Disabling compacts and flushes for region at 1731397860385Disabling writes for close at 1731397860385Obtaining lock to block concurrent updates at 1731397860385Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731397860385Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40436, getHeapSize=51016, getOffHeapSize=0, getCellsCount=127 at 1731397860386 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731397860386Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731397860386Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731397860405 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731397860405Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731397860407 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731397860424 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731397860424Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731397860427 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731397860443 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731397860443Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6af9195: reopening flushed file at 1731397860446 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13436eeb: reopening flushed file at 1731397860448 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34c84c57: reopening flushed file at 1731397860450 (+2 ms)Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=102, compaction requested=false at 1731397860452 (+2 ms)Writing region close event to WAL at 1731397860453 (+1 ms)Closed at 1731397860453 2024-11-12T07:51:00,455 INFO [M:0;f9fc96254ef8:36995 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T07:51:00,455 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:51:00,455 INFO [M:0;f9fc96254ef8:36995 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36995 2024-11-12T07:51:00,455 INFO [M:0;f9fc96254ef8:36995 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:51:00,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:51:00,463 INFO [RS:0;f9fc96254ef8:44587 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:51:00,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44587-0x1012f503c910001, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:51:00,463 INFO [RS:0;f9fc96254ef8:44587 {}] regionserver.HRegionServer(1031): Exiting; stopping=f9fc96254ef8,44587,1731397855844; zookeeper connection closed. 2024-11-12T07:51:00,463 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3ae3b98 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3ae3b98 2024-11-12T07:51:00,463 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-12T07:51:00,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:51:00,578 INFO [M:0;f9fc96254ef8:36995 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:51:00,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36995-0x1012f503c910000, quorum=127.0.0.1:52259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:51:00,580 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T07:51:00,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T07:51:00,581 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:51:00,587 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=69 (was 49) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=401 (was 401), ProcessCount=11 (was 11), AvailableMemoryMB=7050 (was 7235)